neurostats-API 0.0.21b0__py3-none-any.whl → 0.0.23b0__py3-none-any.whl

neurostats_API/fetchers/cash_flow.py

@@ -11,81 +11,69 @@ class CashFlowFetcher(StatsFetcher):
         super().__init__(ticker, db_client)
 
         self.cash_flow_dict = StatsProcessor.load_yaml(
-            "cash_flow_percentage.yaml"
+            "twse/cash_flow_percentage.yaml"
         )  # 計算子表格用
+
+        self.process_function_map = {
+            "twse_stats": self.process_data_twse,
+            "us_stats": self.process_data_us
+        }
     
-    def prepare_query(self, target_season):
+    def prepare_query(self):
         pipeline = super().prepare_query()
 
-        pipeline.append({
-            "$project": {
-                "_id": 0,
-                "ticker": 1,
-                "company_name": 1,
-                "cash_flows": {
-                    "$sortArray": {
-                        "input": {
-                            "$map": {
-                                "input": {
-                                    "$filter": {
-                                        "input": "$seasonal_data",
-                                        "as": "season",
-                                        "cond": {
-                                            "$eq":
-                                            ["$$season.season", target_season]
-                                        }
-                                    }
-                                },
-                                "as": "target_season_data",
-                                "in": {
-                                    "year":
-                                    "$$target_season_data.year",
-                                    "season":
-                                    "$$target_season_data.season",
-                                    "cash_flow":
-                                    "$$target_season_data.cash_flow"
-                                }
-                            }
-                        },
-                        "sortBy": {
-                            "year": -1
-                        }  # 按 year 降序排序
-                    }
+        name_map = {
+            "twse_stats": "cash_flow",
+            "us_stats": "cash_flow"
+        }
+
+
+        chart_name = name_map.get(self.collection_name, "cash_flow")
+
+        append_pipeline = [
+            {
+                "$unwind": "$seasonal_data"    # 展開 seasonal_data 陣列
+            },
+            {
+                "$project": {
+                    "_id": 0,
+                    "ticker": 1,
+                    "company_name": 1,
+                    "year": "$seasonal_data.year",
+                    "season": "$seasonal_data.season",
+                    "cash_flow": {
+                        "$ifNull": [f"$seasonal_data.{chart_name}", []]
+                    }    # 避免 null
+                }
+            },
+            {
+                "$sort": {
+                    "year": -1,
+                    "season": -1
                 }
             }
-        })
+        ]
+
+        pipeline = pipeline + append_pipeline
 
         return pipeline
     
-    def collect_data(self, target_season):
-        pipeline = self.prepare_query(target_season)
-
-        fetched_data = self.collection.aggregate(pipeline)
-
-        return list(fetched_data)[0]
+    def collect_data(self):
+        return super().collect_data()
     
     def query_data(self):
-        
-        try:
-            latest_time = StatsDateTime.get_latest_time(self.ticker, self.collection)['last_update_time']
-            target_season = latest_time['seasonal_data']['latest_season']
-        
-        except:
-            today = StatsDateTime.get_today()
-            target_season = today.season - 1 if (today.season > 1) else 4
-
-        fetched_data = self.collect_data(target_season)
-
-        return self.process_data(fetched_data, target_season)
+        fetched_data = self.collect_data()
+
+        process_fn = self.process_function_map.get(self.collection_name, self.process_data_us)
+        return process_fn(fetched_data)
     
-    def process_data(self, fetched_data, target_season):
+    def process_data_twse(self, fetched_data):
         """
         處理現金流量表頁面的所有表格
         金流表本身沒有比例 但是Ifa有算，
         項目所屬的情況也不一(分別所屬營業,投資,籌資三個活動)
         所以這裡選擇不用slicing處理
         """
-        cash_flows = fetched_data['cash_flows']
 
         index_names = []
         column_names = []
@@ -95,15 +83,7 @@ class CashFlowFetcher(StatsFetcher):
         CASHI_dict = dict()
         CASHF_dict = dict()
 
-        return_dict = {
-            "ticker": fetched_data['ticker'],
-            "company_name": fetched_data['company_name'],
-            "cash_flow": dict(),
-            "CASHO": dict(),
-            "CASHI": dict(),
-            "CASHF": dict()
-        }
-
+        # 處理cash_flow 比例
         checkpoints = ["營業活動之現金流量－間接法", "投資活動之現金流量", "籌資活動之現金流量", "匯率變動對現金及約當現金之影響"]
         main_cash_flows = [
             "營業活動之淨現金流入（流出）", "投資活動之淨現金流入（流出）", "籌資活動之淨現金流入（流出）", None
@@ -112,67 +92,61 @@ class CashFlowFetcher(StatsFetcher):
 
         # 作法: dictionary中也有checkpoints，如果出現了就換下一個index去計算
 
-        for data in cash_flows:
-            year = data['year']
+        for data in fetched_data:
+            year, season, cash_flow = data['year'], data['season'], data['cash_flow']
 
-            time_index = f"{year}Q{target_season}"
+            time_index = f"{year}Q{season}"
 
-            cash_flow = data['cash_flow']
             main_cash_flow_name = None
             partial_cash_flow = None
             next_checkpoint = 0
 
-            for index_name, value in cash_flow.items():
+            temp_dict = {}
+
+            for index_name, cash_flow_value in cash_flow.items():
                 if (next_checkpoint < 3
                         and index_name == checkpoints[next_checkpoint]): # 找到了主要的變動點
                     main_cash_flow_name = main_cash_flows[next_checkpoint]
                     partial_cash_flow = partial_cash_flows[next_checkpoint]
+                    partial_cash_flow[time_index] = {}
                     next_checkpoint += 1
+                    
+                if (isinstance(cash_flow_value, dict)):
+                    value = cash_flow_value.get('value', None) 
+                else:
+                    value = cash_flow_value
+                
+
+                main_value = cash_flow.get(main_cash_flow_name, None)
+                if (isinstance(main_value, dict)):
+                    main_value = main_value.get('value', None)
+                else:
+                    pass
+
                 try:
-                    table_dict[time_index][index_name]['value'] = value[
-                        'value']
-                    if (value['value']):
-                        ratio = np.round(
-                                (value['value'] / cash_flow[
-                                main_cash_flow_name]['value']) * 100, 2)
-                        table_dict[time_index][index_name][
-                            'percentage'] = f"{ratio}%"
-                    else:
-                        table_dict[time_index][index_name][
-                            'percentage'] = None
-                except: # 新增index再做一次
-                    if (time_index not in table_dict.keys()):
-                        table_dict[time_index] = dict()
-                    table_dict[time_index][index_name] = dict()
-
-                    table_dict[time_index][index_name]['value'] = value[
-                        'value']
-                    if (value['value']):
-                        ratio = np.round(
-                                (value['value'] / cash_flow[
-                                main_cash_flow_name]['value']) * 100, 2) 
-                        table_dict[time_index][index_name][
-                            'percentage'] = f"{ratio}%"
-                    else:
-                        table_dict[time_index][index_name][
-                            'percentage'] = None
-                table_dict[time_index][index_name]['value'] = StatsProcessor.cal_non_percentage(value['value'], postfix="千元")
-                try:
-                    partial_cash_flow[time_index][index_name] = table_dict[
-                        time_index][index_name]
+                    ratio = np.round(
+                        (value / main_value) * 100, 2
+                    )
+                    ratio = f"{ratio}%"
                 except:
-                    if (time_index not in partial_cash_flow.keys()):
-                        partial_cash_flow[time_index] = dict()
-                    partial_cash_flow[time_index][index_name] = table_dict[
-                        time_index][index_name]
+                    ratio = None
+                
+                value = StatsProcessor.cal_non_percentage(value, postfix="千元")
+                temp_dict[index_name] = {
+                    "value" : value,
+                    "percentage": ratio
+                }
+                
+                partial_cash_flow[time_index][index_name] = temp_dict[index_name]
 
+            table_dict[time_index] = temp_dict
             index_names += list(cash_flow.keys())
 
         # 轉成dictionary keys
         index_names = list(dict.fromkeys(index_names))
 
         cash_flow_table = pd.DataFrame(table_dict)
-        cash_flow_table = StatsProcessor.expand_value_percentage(cash_flow_table)
+        cash_flow_table_stats = StatsProcessor.expand_value_percentage(cash_flow_table)
 
         CASHO_table = pd.DataFrame(CASHO_dict)
         CASHO_table = StatsProcessor.expand_value_percentage(CASHO_table)
@@ -183,9 +157,44 @@ class CashFlowFetcher(StatsFetcher):
         CASHF_table = pd.DataFrame(CASHF_dict)
         CASHF_table = StatsProcessor.expand_value_percentage(CASHF_table)
 
-        return_dict['cash_flow'] = cash_flow_table
-        return_dict['CASHO'] = CASHO_table
-        return_dict['CASHI'] = CASHI_table
-        return_dict['CASHF'] = CASHF_table
+        for time_index in table_dict.keys():
+            table_dict[time_index] = self.flatten_dict(table_dict[time_index], index_names, target_keys=['value', 'percentage'])
+        cash_flow_flatten = pd.DataFrame.from_dict(table_dict)
+
+        target_season = fetched_data[0]['season']
+        target_season_column = cash_flow_flatten.columns.str.endswith(f"Q{target_season}")
+
+        return_dict = {
+            "ticker": self.ticker,
+            "company_name": fetched_data[-1]['company_name'],
+            "cash_flow": cash_flow_table_stats,
+            "CASHO": CASHO_table,
+            "CASHI": CASHI_table,
+            "CASHF": CASHF_table,
+            "cash_flow_all": cash_flow_flatten,
+            "cash_flow_YoY": cash_flow_flatten.loc[:, target_season_column]
+        }
+        return return_dict
+
+    def process_data_us(self, fetched_data):
+
+        table_dict = {
+            f"{data['year']}Q{data['season']}": data['cash_flow'] 
+            for data in fetched_data
+        }
+
+        cash_flow_df = pd.DataFrame.from_dict(table_dict)
 
+        latest_season = fetched_data[0]['season']
+        target_season_columns = cash_flow_df.columns.str.endswith(
+            f"Q{latest_season}"
+        )
+        cash_flow_df_YoY = cash_flow_df.loc[:, target_season_columns]
+
+        return_dict = {
+            "ticker": self.ticker,
+            "company_name": fetched_data[-1]['company_name'],
+            "cash_flow": cash_flow_df,
+            "cash_flow_YoY": cash_flow_df_YoY
+        }
         return return_dict

neurostats_API/fetchers/finance_overview.py

@@ -16,9 +16,9 @@ class FinanceOverviewFetcher(StatsFetcher):
         super().__init__(ticker, db_client)
 
         self.target_fields = StatsProcessor.load_yaml(
-            "finance_overview_dict.yaml")
+            "twse/finance_overview_dict.yaml")
         self.inverse_dict = StatsProcessor.load_txt(
-            "seasonal_data_field_dict.txt", json_load=True)
+            "twse/seasonal_data_field_dict.txt", json_load=True)
 
     def prepare_query(self, target_year, target_season):
 

neurostats_API/fetchers/month_revenue.py

@@ -149,7 +149,7 @@ class MonthRevenueFetcher(StatsFetcher):
             try:
                 year = data['year'] - 1
                 total = grand_total_dict[year][12]
-                accum_YoY = round((data['grand_total'] / total) * 100, 2)
+                accum_YoY = round(((data['grand_total'] - total) / total) * 100, 2)
                 accum_YoYs.append(f"{accum_YoY}%")
             except Exception as e:
                 accum_YoYs.append(None)

neurostats_API/fetchers/profit_lose.py

@@ -3,11 +3,10 @@ import importlib.resources as pkg_resources
 import json
 import numpy as np
 import pandas as pd
-from ..utils import StatsDateTime, StatsProcessor
+from ..utils import StatsDateTime, StatsProcessor, YoY_Calculator
 import yaml
 
 
-
 class ProfitLoseFetcher(StatsFetcher):
     """
     iFa.ai: 財務分析 -> 損益表
@@ -16,143 +15,219 @@ class ProfitLoseFetcher(StatsFetcher):
     def __init__(self, ticker, db_client):
         super().__init__(ticker, db_client)
 
-        self.table_settings = StatsProcessor.load_yaml("profit_lose.yaml")
+        self.table_settings = StatsProcessor.load_yaml("twse/profit_lose.yaml")
+
+        self.process_function_map = {
+            "twse_stats": self.process_data_twse,
+            "us_stats": self.process_data_us
+        }
 
-    def prepare_query(self, target_season):
+    def prepare_query(self):
         pipeline = super().prepare_query()
 
-        pipeline.append({
-            "$project": {
-                "_id": 0,
-                "ticker": 1,
-                "company_name": 1,
-                "profit_loses": {
-                    "$sortArray": {
-                        "input": {
-                            "$map": {
-                                "input": {
-                                    "$filter": {
-                                        "input": "$seasonal_data",
-                                        "as": "season",
-                                        "cond": {
-                                            "$eq":
-                                            ["$$season.season", target_season]
-                                        }
-                                    }
-                                },
-                                "as": "target_season_data",
-                                "in": {
-                                    "year":
-                                    "$$target_season_data.year",
-                                    "season":
-                                    "$$target_season_data.season",
-                                    "profit_lose":
-                                    "$$target_season_data.profit_lose"
-                                }
-                            }
-                        },
-                        "sortBy": {
-                            "year": -1
-                        }  # 按 year 降序排序
-                    }
+        name_map = {"twse_stats": "profit_lose", "us_stats": "income_statement"}
+
+        chart_name = name_map.get(self.collection_name, "income_statement")
+
+        append_pipeline = [
+            {
+                "$unwind": "$seasonal_data"    # 展開 seasonal_data 陣列
+            },
+            {
+                "$project": {
+                    "_id": 0,
+                    "ticker": 1,
+                    "company_name": 1,
+                    "year": "$seasonal_data.year",
+                    "season": "$seasonal_data.season",
+                    "profit_lose": {
+                        "$ifNull": [f"$seasonal_data.{chart_name}", []]
+                    }    # 避免 null
+                }
+            },
+            {
+                "$sort": {
+                    "year": -1,
+                    "season": -1
                 }
             }
-        })
-
-        return pipeline
+        ]
 
-    def collect_data(self, target_season):
-        pipeline = self.prepare_query(target_season)
+        pipeline = pipeline + append_pipeline
 
-        fetched_data = self.collection.aggregate(pipeline)
+        return pipeline
 
-        return list(fetched_data)[-1]
+    def collect_data(self):
+        return super().collect_data()
 
     def query_data(self):
-        try:
-            latest_time = StatsDateTime.get_latest_time(
-                self.ticker, self.collection)['last_update_time']
-            target_season = latest_time['seasonal_data']['latest_season']
-        except Exception as e:
-            today = StatsDateTime.get_today()
-
-            target_season = today.season
-            target_season = target_season - 1 if target_season > 1 else 4
-
-        fetched_data = self.collect_data(target_season)
+        fetched_data = self.collect_data()
 
-        return self.process_data(fetched_data, target_season)
+        process_fn = self.process_function_map.get(
+            self.collection_name, self.process_data_us
+        )
+        return process_fn(fetched_data)
 
-    def process_data(self, fetched_data, target_season):
+    def process_data_twse(self, fetched_data):
 
-        profit_loses = fetched_data['profit_loses']
+        latest_time = StatsDateTime.get_latest_time(
+            self.ticker, self.collection
+        ).get('last_update_time', {})
 
-        index_names = []
-
-        table_dict = dict()
-        grand_total_dict = dict()
+        # 取最新時間資料時間，沒取到就預設去年年底
+        target_year = latest_time.get('seasonal_data', {}).get(
+            'latest_target_year',
+            StatsDateTime.get_today().year - 1
+        )
+        target_season = latest_time.get(
+            'seasonal_data',{}
+        ).get('latest_season', 4)
 
         return_dict = {
-            "ticker": fetched_data['ticker'],
-            "company_name": fetched_data['company_name'],
+            "ticker": self.ticker,
+            "company_name": fetched_data[-1]['company_name'],
         }
 
-        for data in profit_loses:
-            year = data['year']
-
-            time_index = f"{year}Q{target_season}"
+        profit_lose_dict = {
+            f"{data['year']}Q{data['season']}": data['profit_lose'] 
+            for data in fetched_data
+        }
 
+        profit_lose_df = pd.DataFrame.from_dict(profit_lose_dict)
+        target_season_col = profit_lose_df.columns.str.endswith(
+            f"Q{target_season}"
+        )
+        profit_lose_df = profit_lose_df.loc[:, target_season_col]
+        profit_lose_df = StatsProcessor.expand_value_percentage(
+            profit_lose_df
+        )
+
+        value_col = profit_lose_df.columns.str.endswith(f"_value")
+        percentage_col = profit_lose_df.columns.str.endswith(f"_percentage")
+
+        grand_total_value_col = profit_lose_df.columns.str.endswith(
+            f"grand_total_value"
+        )
+        grand_total_percentage_col = profit_lose_df.columns.str.endswith(
+            f"grand_total_percentage"
+        )
+
+        profit_lose_stats_df = profit_lose_df.loc[:, (
+            (value_col & ~grand_total_value_col) |
+            (percentage_col & ~grand_total_percentage_col)
+        )]
+
+        for time_index, profit_lose in profit_lose_dict.items():
             # 蒐集整體的keys
-            index_names += list(data['profit_lose'].keys())
-            profit_lose = data['profit_lose']
-
-            for index_name, value_dict in profit_lose.items():
-                # (2020Q1, 項目, 金額或%)
-                for item_name, item in value_dict.items():
-                    if ('percentage' in item_name):
-                        if (isinstance(item, (float, int))):
-                            item = StatsProcessor.cal_non_percentage(item, to_str=True, postfix="%")
-                    elif ('YoY' in item_name):
-                        if (isinstance(item, (float, int))):
-                            item = StatsProcessor.cal_percentage(item)
-                    elif ('每股盈餘' in index_name):
-                        if (isinstance(item, (float, int))):
-                            item = StatsProcessor.cal_non_percentage(item,  postfix="元")
-                    else:
-                        if (isinstance(item, (float, int))):
-                            item = StatsProcessor.cal_non_percentage(item,  postfix="千元")
-                    try:
-                        table_dict[index_name][(time_index, item_name)] = item
-
-                    except KeyError:
-                        if (index_name not in table_dict.keys()):
-                            table_dict[index_name] = dict()
-                            grand_total_dict[index_name] = dict()
-
-                        table_dict[index_name][(time_index, item_name)] = item
-
-        total_table = pd.DataFrame.from_dict(table_dict, orient='index')
-        total_table.columns = pd.MultiIndex.from_tuples(total_table.columns)
-
-        total_table = total_table.replace("N/A", None)
+            index_names = list(profit_lose.keys())
+            target_keys = [
+                "value",
+                "percentage",
+                "grand_total",
+                "grand_total_percentage",
+                "YoY_1",
+                "YoY_3",
+                "YoY_5",
+                "YoY_10",
+                "grand_total_YoY_1",
+                "grand_total_YoY_3",
+                "grand_total_YoY_5",
+                "grand_total_YoY_10",
+            ]
+            # flatten dict
+            new_profit_lose = self.flatten_dict(
+                profit_lose, index_names, target_keys
+            )
+            profit_lose_dict[time_index] = new_profit_lose
+
+        profit_lose_df = pd.DataFrame.from_dict(profit_lose_dict)
+
+        # EPS的value用元計算
+        eps_index = profit_lose_df.index.str.endswith(
+            "_value"
+        ) & profit_lose_df.index.str.contains("每股盈餘")
+        profit_lose_df.loc[eps_index] = profit_lose_df.loc[
+            eps_index].apply(
+                lambda x: StatsProcessor.cal_non_percentage(x, postfix="元")
+            )
+
+        # percentage處理
+        percentage_index = profit_lose_df.index.str.endswith("percentage")
+        profit_lose_df.loc[percentage_index] = profit_lose_df.loc[
+            percentage_index].apply(
+                lambda x: StatsProcessor.
+                cal_non_percentage(x, to_str=True, postfix="%")
+            )
+
+        # YoY處理: 乘以100
+        YoY_index = profit_lose_df.index.str.contains("YoY")
+        profit_lose_df.loc[YoY_index] = profit_lose_df.loc[
+            YoY_index].apply(lambda x: StatsProcessor.cal_percentage(x))
+
+        # 剩下的處理: 乘以千元
+        value_index = ~(
+            percentage_index | YoY_index | profit_lose_df.index.isin(eps_index)
+        )    # 除了上述以外的 index
+        profit_lose_df.loc[value_index] = profit_lose_df.loc[
+            value_index].apply(
+                lambda x: StatsProcessor.cal_non_percentage(x, postfix="千元")
+            )
+
+        total_table = profit_lose_df.replace("N/A", None)
+
+        # 取特定季度
+        target_season_columns = total_table.columns.str.endswith(
+            f"Q{target_season}"
+        )
+        total_table_YoY = total_table.loc[:, target_season_columns]
 
         for name, setting in self.table_settings.items():
-            if ('target_index' in setting.keys()):
-                target_indexes = [target.strip() for target in setting['target_index']]
-            else:
-                target_indexes = [None]
-            
+            target_indexes = setting.get('target_index', [None])
             for target_index in target_indexes:
                 try:
-                    return_dict[name] = StatsProcessor.slice_multi_col_table(
-                        total_table=total_table,
+                    return_dict[name] = StatsProcessor.slice_table(
+                        total_table=total_table_YoY,
                         mode=setting['mode'],
-                        target_index=target_index)
+                        target_index=target_index
+                    )
                     break
                 except Exception as e:
-                    return_dict[name] = StatsProcessor.slice_multi_col_table(
-                        total_table=total_table,
-                        mode=setting['mode'],
-                        target_index=target_index)
+                    print(str(e))
+                    continue
+
+        return_dict.update(
+            {
+                "profit_lose": profit_lose_stats_df,
+                "profit_lose_all": total_table.copy(),
+                "profit_lose_YoY": total_table_YoY
+            }
+        )
+        return return_dict
+
+    def process_data_us(self, fetched_data):
+
+        table_dict = {
+            f"{data['year']}Q{data['season']}": data['profit_lose']
+            for data in fetched_data
+        }
+
+        table_dict = YoY_Calculator.cal_QoQ(table_dict)
+        table_dict = YoY_Calculator.cal_YoY(table_dict)
+
+        for time_index, data_dict in table_dict.items():
+            table_dict[time_index] = self.flatten_dict(
+                value_dict=data_dict,
+                indexes=list(data_dict.keys()),
+                target_keys=["value", "growth"] + 
+                            [f"YoY_{i}" for i in [1, 3, 5, 10]]
+            )
+
+        # 計算QoQ
+
+        return_dict = {
+            "ticker": self.ticker,
+            "company_name": fetched_data[-1]['company_name'],
+            "profit_lose": pd.DataFrame.from_dict(table_dict)
+        }
 
         return return_dict