neurostats-API 0.0.25rc1__py3-none-any.whl → 1.0.0__py3-none-any.whl

neurostats_API/{tools → config}/tej_db/tej_db_skip_index.yaml

@@ -8,7 +8,6 @@ TWN/AINVFQ1:
 - curr
 - annd
 - fin_ind
-- eps
 - r307
 - r305
 - r306
@@ -26,7 +25,6 @@ TWN/AFESTM1:
 - curr
 - annd
 - fin_ind
-- eps
 - r307
 - r305
 - r306

neurostats_API/{tools → config}/twse/profit_lose.yaml

@@ -1,9 +1,3 @@
-profit_lose:  # 總營收表
-  mode: value_and_percentage
-
-grand_total_profit_lose:
-  mode: grand_total_values
-  
 revenue:
   mode: growth
   target_index: 

neurostats_API/fetchers/finance_overview.py

@@ -808,6 +808,7 @@ class AgentFinanceOverviewFetcher(FinanceOverviewFetcher):
         self.us_company_list = StatsProcessor.load_json("company_list/us_TradingView_list.json")
 
         if (ticker not in self.us_company_list.keys()):
+            # 目前只支援美股
             raise NoCompanyError("class \"AgentFinanceOverviewFetcher\" only supports US company now")
 
         db_name = "company" if self.ticker in self.tw_company_list else "company_us"
@@ -835,7 +836,6 @@ class AgentFinanceOverviewFetcher(FinanceOverviewFetcher):
             self.seasonal_collection_name]
 
     def query_data(self, date=None):
-
         query_set = self._prepare_query(date)
         if (date is None):
             str_date = datetime.today().strftime("%Y-%m-%d")
@@ -948,6 +948,9 @@ class AgentFinanceOverviewFetcher(FinanceOverviewFetcher):
             return "Error"
 
     def _get_latest_average_volume(self, daily_data, length=30):
+        """
+        計算平均成交量
+        """
         try:
             daily_data = pd.DataFrame(daily_data).tail(length)
 
@@ -960,6 +963,10 @@ class AgentFinanceOverviewFetcher(FinanceOverviewFetcher):
             return "Error"
 
     def _get_market_capitalization(self, daily_data, seasonal_data):
+        """
+        計算市值
+        市值 = 收盤價 * 流通股數
+        """
         try:
             latest_close = float(self._get_latest_close(daily_data))
             latest_common_share = seasonal_data[-1][
@@ -973,6 +980,10 @@ class AgentFinanceOverviewFetcher(FinanceOverviewFetcher):
             return "Error"
 
     def _get_PE(self, daily_data, TTM_data):
+        """
+        本益比
+        本益比 = 收盤價 / 稀釋EPS(TTM)
+        """
         try:
             latest_close = float(self._get_latest_close(daily_data))
             TTM_EPS = TTM_data['Diluted EPS']
@@ -985,6 +996,10 @@ class AgentFinanceOverviewFetcher(FinanceOverviewFetcher):
             return "Error"
 
     def _get_PS(self, daily_data, TTM_data, seasonal_data):
+        """
+        股價營收比: 
+        本益比 = 市值 / 總營收(TTM)
+        """
         try:
             market_capitalzation = self._get_market_capitalization(daily_data, seasonal_data)
             market_capitalzation = float(market_capitalzation)
@@ -997,7 +1012,9 @@ class AgentFinanceOverviewFetcher(FinanceOverviewFetcher):
             return "Error"
 
     def _is_us_market_open(self):
-        
+        """
+        判斷目前美股是否開市
+        """
         taiwan_dt = datetime.now(pytz.timezone('Asia/Taipei'))
 
         # 轉成美東時間（會自動處理夏令時間）
@@ -1020,6 +1037,9 @@ class AgentFinanceOverviewFetcher(FinanceOverviewFetcher):
         return market_open <= us_dt <= market_close
     
     def _get_category(self):
+        """
+        判斷公司類別，透過檔案
+        """
         category = self.us_company_list.get(self.ticker,{}).get('en_industry')
 
         if (category):

neurostats_API/transformers/__init__.py

@@ -0,0 +1,40 @@
+from .balance_sheet import (
+    TWSEBalanceSheetTransformer,
+    USBalanceSheetTransformer
+)
+
+from .cash_flow import (
+    TWSECashFlowTransformer,
+    USCashFlowTransformer
+)
+
+from .daily_tech import(
+    DailyTechTransformer
+)
+
+from .daily_chip import(
+    TWSEChipTransformer
+)
+
+from .month_revenue import(
+    TWSEMonthlyRevenueTransformer
+)
+
+from .profit_lose import (
+    TWSEProfitLoseTransformer,
+    USProfitLoseTransformer
+)
+
+from .tej import (
+    TEJFinanceStatementTransformer
+)
+
+from .value import (
+    TWSEAnnualValueTransformer,
+    TWSEHistoryValueTransformer
+)
+
+from .finance_overview import (
+    AgentOverviewTransformer,
+    FinanceOverviewTransformer
+)

neurostats_API/transformers/balance_sheet/__init__.py

@@ -0,0 +1,2 @@
+from .twse import TWSEBalanceSheetTransformer
+from .us import USBalanceSheetTransformer

neurostats_API/transformers/balance_sheet/base.py

@@ -0,0 +1,51 @@
+from  ..base import BaseTransformer
+import pandas as pd
+
+class BaseBalanceSheetTransformer(BaseTransformer):
+    def __init__(self, ticker, company_name, zone):
+        super().__init__(ticker, company_name, zone)
+
+    def _process_twse_to_stats_format(self, fetched_data):
+        table_dict = {}
+        for data in fetched_data:
+            year, season, balance_sheet = data['year'], data['season'], data['balance_sheet']
+
+            time_index = f"{year}Q{season}"
+            table_dict[time_index] = balance_sheet
+
+        stats_df = pd.DataFrame(table_dict)
+
+        return stats_df
+    
+    def _process_twse_to_tej_format(self, fetched_data):
+        """
+        new_df 迎合 TEJ格式, 用於 report_generator
+        """
+        table_dict = {}
+        for data in fetched_data:
+            year, season, balance_sheet = data['year'], data['season'], data['balance_sheet']
+            time_index = f"{year}Q{season}"
+
+            new_balance_sheet = self.flatten_dict(
+                balance_sheet,
+                target_keys=["value", "percentage"]
+            )
+
+            table_dict[time_index] = new_balance_sheet
+        
+        stats_df = pd.DataFrame.from_dict(table_dict)
+        return stats_df.T
+    
+    def _process_us_format(self, fetched_data):
+        """
+        主要用於report generator
+        """
+        table_dict = {}
+        for data in fetched_data:
+            year, season, balance_sheet = data['year'], data['season'], data['balance_sheet']
+            time_index = f"{year}Q{season}"
+
+            table_dict[time_index] = balance_sheet
+        
+        stats_df = pd.DataFrame.from_dict(table_dict)
+        return stats_df.T

neurostats_API/transformers/balance_sheet/twse.py

@@ -0,0 +1,81 @@
+from .base import BaseBalanceSheetTransformer
+from neurostats_API.utils import StatsProcessor
+
+
+class TWSEBalanceSheetTransformer(BaseBalanceSheetTransformer):
+
+    def __init__(self, ticker, company_name, zone):
+        super().__init__(ticker, company_name, zone)
+
+        self.table_settings = StatsProcessor.load_yaml(
+            "twse/balance_sheet.yaml"
+        )
+        self.return_keys = [
+            'balance_sheet', 'total_asset', 'current_asset',
+            'non_current_asset', 'current_debt', 'non_current_debt', 'equity',
+            'balance_sheet_all', 'balance_sheet_YoY'
+        ]
+
+        self.stats_df = None
+        self.new_df = None
+
+    def process_transform(self, fetched_data):
+        if (not fetched_data):
+            return self._get_empty_structure()
+
+        processed_data = self._process_fn(fetched_data)
+
+        return processed_data
+
+    def _process_fn(self, fetched_data):
+
+        return_dict = {"ticker": self.ticker, "company_name": self.company_name}
+
+        target_season = fetched_data[0]['season']
+
+        self.stats_df = self._process_twse_to_stats_format(fetched_data)
+        self.stats_df = self._slice_target_season(self.stats_df, target_season)
+
+        # 轉換格式 (元 / 千元 / %)
+        self.stats_df = StatsProcessor.expand_value_percentage(self.stats_df)
+        self.stats_df = self._apply_process_unit_pipeline(
+            self.stats_df, postfix_list=['_value', "_percentage"]
+        )
+
+        self.new_df = self._process_twse_to_tej_format(fetched_data)
+        # 轉換格式 (元 / 千元 / %)
+        self.new_df = self._apply_process_unit_pipeline(
+            self.new_df, postfix_list=['_value', "_percentage"]
+        )
+
+        self.new_dict = self.new_df.to_dict()
+        self.new_df = self._cal_QoQ(self.new_dict)
+        self.new_df = self.new_df.T
+
+        total_table_YoY = self._slice_target_season(
+            self.new_df, target_season
+        )
+
+        return_dict.update(
+            {
+                "balance_sheet": self.stats_df,
+                "balance_sheet_all": self.new_df,
+                "balance_sheet_YoY": total_table_YoY
+            }
+        )
+        # 抽取出小欄位整合
+        self._process_target_columns(return_dict, self.stats_df)
+
+        return return_dict
+
+    def _process_target_columns(self, return_dict, stats_df):
+        for name, setting in self.table_settings.items():
+            target_indexes = setting.get('target_index', [None])
+            for target_index in target_indexes:
+                try:
+                    return_dict[name] = StatsProcessor.slice_old_table(
+                        total_table=stats_df, target_index=target_index
+                    )
+                    break
+                except Exception as e:
+                    continue

neurostats_API/transformers/balance_sheet/us.py

@@ -0,0 +1,38 @@
+from .base import BaseBalanceSheetTransformer
+from neurostats_API.utils import StatsProcessor
+
+
+class USBalanceSheetTransformer(BaseBalanceSheetTransformer):
+
+    def __init__(self, ticker, company_name, zone):
+        super().__init__(ticker, company_name, zone)
+
+        self.data_df = None
+        self.return_keys = [
+            'balance_sheet',
+            'balance_sheet_YoY'
+        ]
+
+    def process_transform(self, fetched_data):
+        if (not fetched_data):
+            return self._get_empty_structure()
+
+        return_dict = {"ticker": self.ticker, "company_name": self.company_name}
+
+        # QoQ表格
+        self.data_df = self._process_us_format(fetched_data)
+
+        # YoY表格
+        target_season = fetched_data[0]['season']
+        total_table_YoY = self._slice_target_season(
+            self.data_df.T, target_season
+        )
+
+        return_dict.update(
+            {
+                'balance_sheet': self.data_df,
+                "balance_sheet_YoY": total_table_YoY
+            }
+        )
+
+        return return_dict

neurostats_API/transformers/base.py

@@ -0,0 +1,158 @@
+import abc
+from neurostats_API.utils import StatsProcessor, YoY_Calculator
+import pandas as pd
+
+
+class BaseTransformer(abc.ABC):
+    """
+    Transformer用途: 轉換資料為fetcher可使用的格式
+    """
+
+    def __init__(self, ticker, company_name, zone):
+        self.ticker = ticker
+        self.company_name = company_name
+
+        self.zone = zone
+        self.return_keys = []
+
+    @abc.abstractmethod
+    def process_transform(self):
+        pass
+
+    @classmethod
+    def flatten_dict(cls, value_dict, target_keys):
+        indexes = value_dict.keys()
+        new_dict = {}
+
+        for key in indexes:
+            new_dict.update(
+                {
+                    f"{key}_{sub_key}": value_dict[key].get(sub_key, None)
+                    for sub_key in target_keys
+                }
+            )
+        
+        return new_dict
+
+    @staticmethod
+    def _process_unit(data_df, postfix):
+
+        lambda_map = {
+            "_value":
+            lambda x: StatsProcessor.cal_non_percentage(x, postfix="千元"),
+            "_percentage":
+            lambda x: StatsProcessor.
+            cal_non_percentage(x, to_str=True, postfix="%"),
+            '_growth':
+            lambda x: StatsProcessor.
+            cal_non_percentage(x, to_str=True, postfix="%"),
+            "_YoY_1":
+            StatsProcessor.cal_percentage,
+            "_YoY_3":
+            StatsProcessor.cal_percentage,
+            "_YoY_5":
+            StatsProcessor.cal_percentage,
+            "_YoY_10":
+            StatsProcessor.cal_percentage
+        }
+
+        process_fn = lambda_map.get(postfix)
+        postfix_cols = data_df.columns.str.endswith(postfix)
+        postfix_cols = data_df.loc[:, postfix_cols].columns
+
+        if (postfix == "_value"):
+            postfix_cols = [
+                col for col in postfix_cols
+                if not ("eps" in col or "每股盈餘" in col)
+            ]
+
+        if (postfix == '_growth'):
+            data_df[postfix_cols] = data_df[postfix_cols].map(
+                lambda x: x * 100.0 if isinstance(x, float) else x
+            )
+
+        data_df[postfix_cols] = (data_df[postfix_cols].map(process_fn))
+
+        return data_df
+
+    def _apply_process_unit_pipeline(
+        self, data_df, postfix_list=["_value", "percentage"]
+    ):
+        for postfix in postfix_list:
+            data_df = self._process_unit(data_df, postfix)
+        return data_df
+
+    @staticmethod
+    def _slice_target_season(stats_df, target_season):
+        target_season_columns = stats_df.columns.str.endswith(
+            f"Q{target_season}"
+        )
+        stats_df = stats_df.loc[:, target_season_columns]
+
+        return stats_df
+
+    def _get_empty_structure(self):
+        return_dict = {
+            "warning": "No data fetched",
+            "ticker": self.ticker,
+            "company_name": self.company_name
+        }
+
+        return_dict.update(
+            {
+                key: pd.DataFrame(columns=pd.Index([], name='index'))
+                for key in self.return_keys
+            }
+        )
+        return return_dict
+
+    def _process_value(self, value):
+        if isinstance(value, str) and "%" in value:
+            value = value.replace("%", "")
+        try:
+            return float(value)
+        except (ValueError, TypeError):
+            return None
+
+    def _calculate_growth(self, this_value, last_value, delta):
+        try:
+            return YoY_Calculator.cal_growth(
+                this_value, last_value, delta
+            ) * 100
+        except Exception:
+            return None
+
+    def _cal_QoQ(self, data_dict):
+        """
+        data_dict: {"<key>_value": {"2020Q1": <value>, ....}, "<key>_percentage": {"2020Q1": <value>, ....}}
+        """
+
+        return_dict = data_dict.copy()
+
+        for key, datas in data_dict.items():
+            if (key.endswith("_value")):
+                main_key = key.split("_")[0]
+                temp_growth_dict = dict()
+                for year_season, value in datas.items():
+
+                    year, season = year_season.split('Q')
+                    year, season = int(year), int(season)
+
+                    last_year, last_season = (
+                        year, season - 1
+                    ) if season != 1 else (year - 1, 4)
+
+                    last_value = datas.get(f"{last_year}Q{last_season}", None)
+
+                    growth = YoY_Calculator.cal_growth(
+                        value, last_value, delta=1
+                    )
+                    if (growth):
+                        growth = growth * 100
+                        growth = f"{growth:.2f}%"
+
+                    temp_growth_dict[year_season] = growth
+
+                return_dict[f"{main_key}_growth"] = growth
+        return_df = pd.DataFrame.from_dict(return_dict)
+        return return_df

neurostats_API/transformers/cash_flow/__init__.py

@@ -0,0 +1,2 @@
+from .twse import TWSECashFlowTransformer
+from .us import USCashFlowTransformer

neurostats_API/transformers/cash_flow/base.py

@@ -0,0 +1,114 @@
+from ..base import BaseTransformer
+from neurostats_API.utils import StatsProcessor
+import numpy as np
+import pandas as pd
+
+class BaseCashFlowTransformer(BaseTransformer):
+    def __init__(self, ticker, company_name, zone):
+        super().__init__(ticker, company_name, zone)
+    
+    def _process_twse_to_stats_format(self, fetched_data):
+        table_dict = {}
+        CASHO_dict = {}
+        CASHI_dict = {}
+        CASHF_dict = {}
+
+        checkpoints = ["營業活動之現金流量－間接法", "投資活動之現金流量", "籌資活動之現金流量", "匯率變動對現金及約當現金之影響"]
+        main_cash_flow_names = [
+            "營業活動之淨現金流入（流出）", "投資活動之淨現金流入（流出）", "籌資活動之淨現金流入（流出）", "其他"
+        ] # 主要的比例對象
+        partial_cash_flow_dicts = [CASHO_dict, CASHI_dict, CASHF_dict, dict()] 
+
+        boundary = len(main_cash_flow_names) - 1
+        
+        for data in fetched_data:
+            year, season, cash_flow = data['year'], data['season'], data['cash_flow']
+
+            time_index = f"{year}Q{season}"
+            table_dict[time_index] = cash_flow
+
+            # 處理sub part
+            main_cash_flow_name = ""
+            partial_cash_flow = {}
+            next_checkpoint = 0
+
+            temp_dict = {}
+
+            for index_name, cash_flow_value in cash_flow.items():
+                if (next_checkpoint < boundary
+                        and index_name == checkpoints[next_checkpoint]): # 找到了主要的變動點
+                    main_cash_flow_name = main_cash_flow_names[next_checkpoint]
+                    partial_cash_flow = partial_cash_flow_dicts[next_checkpoint]
+                    partial_cash_flow[time_index] = {}
+                    next_checkpoint += 1
+
+                if main_cash_flow_name: # 有取得cash flow name再進行接下來的計算percentage
+                    if (isinstance(cash_flow_value, dict)):
+                        value = cash_flow_value.get('value', None) 
+                    else:
+                        value = cash_flow_value
+
+                    # 處理cashflow percentage部分(取2位 + 接%)
+                    main_value = cash_flow.get(main_cash_flow_name, None)
+                    if (isinstance(main_value, dict)):
+                        main_value = main_value.get('value', None)
+                    else:
+                        pass
+
+                    try:
+                        ratio = np.round(
+                            (value / main_value) * 100, 2
+                        )
+                        ratio = f"{ratio}%"
+                    except:
+                        ratio = None
+                    
+                    value = StatsProcessor.cal_non_percentage(value, postfix="千元")
+                    temp_dict[index_name] = {
+                        "value" : value,
+                        "percentage": ratio
+                    }
+                    partial_cash_flow[time_index][index_name] = temp_dict[index_name]
+
+            table_dict[time_index] = temp_dict
+
+        stats_df = pd.DataFrame(table_dict)
+
+        CASHO_df = pd.DataFrame(CASHO_dict)
+        CASHI_df = pd.DataFrame(CASHI_dict)
+        CASHF_df = pd.DataFrame(CASHF_dict)
+
+        return stats_df, CASHO_df, CASHI_df, CASHF_df
+    
+    def _process_twse_to_tej_format(self, fetched_data):
+        """
+        new_df 迎合 TEJ格式, 用於 report_generator
+        """
+        table_dict = {}
+        for data in fetched_data:
+            year, season, cash_flow = data['year'], data['season'], data['cash_flow']
+            time_index = f"{year}Q{season}"
+
+            new_cash_flow = self.flatten_dict(
+                cash_flow,
+                target_keys=["value", "percentage"]
+            )
+
+            table_dict[time_index] = new_cash_flow
+        
+        stats_df = pd.DataFrame.from_dict(table_dict)
+        return stats_df
+    
+    def _process_us_format(self, fetched_data):
+        """
+        主要用於report generator
+        """
+        table_dict = {}
+        for data in fetched_data:
+            year, season, cash_flow = data['year'], data['season'], data['cash_flow']
+            time_index = f"{year}Q{season}"
+
+            table_dict[time_index] = cash_flow
+        
+        stats_df = pd.DataFrame.from_dict(table_dict)
+        return stats_df

neurostats_API/transformers/cash_flow/twse.py

@@ -0,0 +1,70 @@
+from .base import BaseCashFlowTransformer
+from neurostats_API.utils import StatsProcessor
+
+class TWSECashFlowTransformer(BaseCashFlowTransformer):
+    def __init__(self, ticker, company_name, zone):
+        super().__init__(ticker, company_name, zone)
+
+        self.cash_flow_dict = StatsProcessor.load_yaml(
+            "twse/cash_flow_percentage.yaml"
+        )  # 計算子表格用
+
+        self.return_keys = ['cash_flow', 'CASHO', 'CASHI', 'CASHF', 'cash_flow_all', 'cash_flow_YoY']
+
+    def process_transform(self, fetched_data):
+        """
+        轉換整體
+        """
+        if (not fetched_data):
+            return self._get_empty_structure()
+
+        return_dict = {
+            "ticker": self.ticker,
+            "company_name": self.company_name
+        }
+
+        return_dict.update(self.process_stats_page(fetched_data))
+        return_dict.update(self.process_QoQ(fetched_data))
+
+        return return_dict
+    def process_stats_page(self, fetched_data):
+        """
+        這裡主要回傳stats頁面相關的部分
+        """
+        if (not fetched_data):
+            return self._get_empty_structure()
+
+        stats_page_df, CASHO_df, CASHI_df, CASHF_df = self._process_twse_to_stats_format(fetched_data)
+        target_season = fetched_data[0]['season']
+
+        return {
+            'cash_flow': self._slice_and_expand(stats_page_df, target_season),
+            "CASHO": self._slice_and_expand(CASHO_df, target_season),
+            "CASHI": self._slice_and_expand(CASHI_df, target_season),
+            "CASHF": self._slice_and_expand(CASHF_df, target_season)
+        }
+    
+    def process_QoQ(self, fetched_data):
+        """
+        這裡主要只會回傳其他地方需要的部分，此符合TEJ格式
+        """
+        if (not fetched_data):
+            return self._get_empty_structure()
+        
+        data_df = self._process_twse_to_tej_format(fetched_data)
+        data_df = self._cal_QoQ(data_df.T.to_dict())
+        data_df = data_df.T
+        target_season = fetched_data[0]['season']
+
+        data_df_YoY = self._slice_target_season(data_df, target_season)
+
+        return {
+            "cash_flow_all": data_df,
+            "cash_flow_YoY": data_df_YoY
+        }
+
+    def _slice_and_expand(self, data_df, target_season):
+        data_df = self._slice_target_season(data_df, target_season)
+        data_df = StatsProcessor.expand_value_percentage(data_df)
+
+        return data_df