neurostats-API 0.0.22__py3-none-any.whl → 0.0.23__py3-none-any.whl

neurostats_API/fetchers/margin_trading.py

@@ -1,5 +1,5 @@
 from .base import StatsFetcher
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date
 import json
 import numpy as np
 import pandas as pd
@@ -23,70 +23,84 @@ class MarginTradingFetcher(StatsFetcher):
     def prepare_query(self, start_date, end_date):
         pipeline = super().prepare_query()
 
-        pipeline.append({
-            "$project": {
-                "_id": 0,
-                "ticker": 1,
-                "company_name": 1,
-                "daily_data": {
-                    "$map": {
-                        "input": {
-                            "$filter": {
-                                "input": "$daily_data",
-                                "as": "daliy",
-                                "cond": {
-                                    "$and": [{
-                                        "$gte": ["$$daliy.date", start_date]
-                                    }, {
-                                        "$lte": ["$$daliy.date", end_date]
-                                    }]
+        pipeline.append(
+            {
+                "$project": {
+                    "_id": 0,
+                    "ticker": 1,
+                    "company_name": 1,
+                    "daily_data": {
+                        "$map": {
+                            "input": {
+                                "$filter": {
+                                    "input": "$daily_data",
+                                    "as": "daliy",
+                                    "cond": {
+                                        "$and": [
+                                            {
+                                                "$gte":
+                                                ["$$daliy.date", start_date]
+                                            }, {
+                                                "$lte":
+                                                ["$$daliy.date", end_date]
+                                            }
+                                        ]
+                                    }
                                 }
-                            }
-                        },
-                        "as": "target_daliy_data",
-                        "in": "$$target_daliy_data"
-                    }
-                },
-                "margin_trading": {
-                    "$map": {
-                        "input": {
-                            "$filter": {
-                                "input": "$margin_trading",
-                                "as": "margin",
-                                "cond": {
-                                    "$and": [{
-                                        "$gte": ["$$margin.date", start_date]
-                                    }, {
-                                        "$lte": ["$$margin.date", end_date]
-                                    }]
+                            },
+                            "as": "target_daliy_data",
+                            "in": "$$target_daliy_data"
+                        }
+                    },
+                    "margin_trading": {
+                        "$map": {
+                            "input": {
+                                "$filter": {
+                                    "input": "$margin_trading",
+                                    "as": "margin",
+                                    "cond": {
+                                        "$and": [
+                                            {
+                                                "$gte":
+                                                ["$$margin.date", start_date]
+                                            }, {
+                                                "$lte":
+                                                ["$$margin.date", end_date]
+                                            }
+                                        ]
+                                    }
                                 }
-                            }
-                        },
-                        "as": "target_margin_data",
-                        "in": "$$target_margin_data"
-                    }
-                },
-                "security_lending": {
-                    "$map": {
-                        "input": {
-                            "$filter": {
-                                "input": "$security_lending",
-                                "as": "lending",
-                                "cond": {
-                                    "$and": [{
-                                        "$gte": ["$$lending.date", start_date]
-                                    }, {
-                                        "$lte": ["$$lending.date", end_date]
-                                    }]
+                            },
+                            "as": "target_margin_data",
+                            "in": "$$target_margin_data"
+                        }
+                    },
+                    "security_lending": {
+                        "$map": {
+                            "input": {
+                                "$filter": {
+                                    "input": "$security_lending",
+                                    "as": "lending",
+                                    "cond": {
+                                        "$and": [
+                                            {
+                                                "$gte":
+                                                ["$$lending.date", start_date]
+                                            }, {
+                                                "$lte":
+                                                ["$$lending.date", end_date]
+                                            }
+                                        ]
+                                    }
                                 }
-                            }
-                        },
-                        "as": "target_lending_data",
-                        "in": "$$target_lending_data"
+                            },
+                            "as": "target_lending_data",
+                            "in": "$$target_lending_data"
+                        }
                     }
                 }
             }
-        })
+        )
 
         return pipeline
 
@@ -100,39 +114,39 @@ class MarginTradingFetcher(StatsFetcher):
     def query_data(self):
         try:
             latest_time = StatsDateTime.get_latest_time(
-                self.ticker, self.collection)['last_update_time']
+                self.ticker, self.collection
+            )['last_update_time']
             latest_date = latest_time['margin_trading']['latest_date']
-            end_date = latest_date.replace(hour=0,
-                                           minute=0,
-                                           second=0,
-                                           microsecond=0)
-        except Exception as e:
-            print(
-                f"No updated time for institution_trading in {self.ticker}, use current time instead"
+            end_date = latest_date.replace(
+                hour=0, minute=0, second=0, microsecond=0
             )
+        except Exception as e:
             end_date = datetime.now(self.timezone)
-            end_date = end_date.replace(hour=0,
-                                        minute=0,
-                                        second=0,
-                                        microsecond=0)
+            end_date = end_date.replace(
+                hour=0, minute=0, second=0, microsecond=0
+            )
 
-            if (end_date.hour < 22):  # 拿不到今天的資料
+            if (end_date.hour < 22):    # 拿不到今天的資料
                 end_date = end_date - timedelta(days=1)
 
         start_date = end_date - timedelta(days=365)
 
         fetched_data = self.collect_data(start_date, end_date)
 
-        fetched_data['daily_data'] = sorted(fetched_data['daily_data'],
-                                            key=lambda x: x['date'],
-                                            reverse=True)
-        fetched_data['margin_trading'] = sorted(fetched_data['margin_trading'],
-                                                key=lambda x: x['date'],
-                                                reverse=True)
+        fetched_data['daily_data'] = sorted(
+            fetched_data['daily_data'], key=lambda x: x['date'], reverse=True
+        )
+        fetched_data['margin_trading'] = sorted(
+            fetched_data['margin_trading'],
+            key=lambda x: x['date'],
+            reverse=True
+        ) if (fetched_data['margin_trading']) else []
+
         fetched_data['security_lending'] = sorted(
             fetched_data['security_lending'],
             key=lambda x: x['date'],
-            reverse=True)
+            reverse=True
+        )if (fetched_data['security_lending']) else []
 
         table_dict = self.process_data(fetched_data)
 
@@ -149,19 +163,18 @@ class MarginTradingFetcher(StatsFetcher):
         price_dict = {
             "open": latest_data['open'],
             'close': latest_data['close'],
-            'range': f"{latest_data['low']} - {latest_data['high']}",
-            'volume': latest_data['volume'] / 1000,
+            'range': f"{latest_data['low']:.2f} - {latest_data['high']:.2f}",
+            'volume': round(float(latest_data['volume']) / 1000, 2),
             'last_open': yesterday_data['open'],
             'last_close': yesterday_data['close'],
-            'last_range':
-            f"{yesterday_data['low']} - {yesterday_data['high']}",
-            'last_volume': yesterday_data['volume'] / 1000
+            'last_range': f"{yesterday_data['low']:.2f} - {yesterday_data['high']:.2f}",
+            'last_volume': round(float(yesterday_data['volume']) / 1000, 2)
         }
         annual_lows = [data['low'] for data in daily_datas]
         annual_highs = [data['high'] for data in daily_datas]
         lowest = np.min(annual_lows).item()
         highest = np.max(annual_highs).item()
-        price_dict['52weeks_range'] = f"{lowest} - {highest}"
+        price_dict['52weeks_range'] = f"{lowest:.2f} - {highest:.2f}"
 
         return_dict['price'] = price_dict
 
@@ -169,6 +182,18 @@ class MarginTradingFetcher(StatsFetcher):
         margin_trading = fetched_data['margin_trading']
         security_lending = fetched_data['security_lending']
 
+        return_dict['margin_trading'] = pd.DataFrame()
+        return_dict['stock_lending'] = pd.DataFrame()
+        return_dict['latest_trading'] = {
+            'date': date.today(),
+            "margin_trading": pd.DataFrame(),
+            "stock_lending": pd.DataFrame()
+        }
+        return_dict['annual_margin'] = pd.DataFrame()
+        return_dict['security_offset'] = 0.0
+        if (not margin_trading):
+            return return_dict
+
         latest_margin_date = margin_trading[0]['date']
         latest_lending_date = security_lending[0]['date']
         ## 融資融券
@@ -176,7 +201,8 @@ class MarginTradingFetcher(StatsFetcher):
         for trading in margin_trading:
             trading['financing']['現償'] = trading['financing'].pop('現金償還')
             trading['short_selling']['現償'] = trading['short_selling'].pop(
-                '現券償還')
+                '現券償還'
+            )
         ### 轉換
         latest_margin_trading = margin_trading[0]
         latest_margin_trading_df = {
@@ -185,7 +211,8 @@ class MarginTradingFetcher(StatsFetcher):
             if (isinstance(sub_dict, dict))
         }
         latest_margin_trading_df = pd.DataFrame.from_dict(
-            latest_margin_trading_df)
+            latest_margin_trading_df
+        )
 
         ## 借券表格
         latest_stock_lending = security_lending[0]['stock_lending']
@@ -195,9 +222,9 @@ class MarginTradingFetcher(StatsFetcher):
             for type_name, value in latest_stock_lending.items()
         }
         latest_stock_lending.pop("前日餘額")
-        latest_stock_lending_df = pd.DataFrame.from_dict(latest_stock_lending,
-                                                         orient="index",
-                                                         columns=['stock_lending'])
+        latest_stock_lending_df = pd.DataFrame.from_dict(
+            latest_stock_lending, orient="index", columns=['stock_lending']
+        )
 
         latest_dict = {
             "date": latest_margin_date,
@@ -262,14 +289,14 @@ class MarginTradingFetcher(StatsFetcher):
 
         annual_dict = {
             date: {
-                "close": close_prices[date],
-                "volume": volumes[date],
+                "close": close_prices.get(date, 0.0),
+                "volume": volumes.get(date, 0.0),
                 **financings[date],
                 **short_sellings[date],
                 **stock_lendings[date], 
-                "資券互抵":security_offsets[date]
+                "資券互抵": security_offsets[date]
             }
-            for date in annual_dates
+            for date in financings.keys()
         }
 
         annual_table = pd.DataFrame.from_dict(annual_dict)

neurostats_API/fetchers/month_revenue.py

@@ -4,6 +4,8 @@ import pandas as pd
 from ..utils import StatsDateTime, StatsProcessor, YoY_Calculator
 import importlib.resources as pkg_resources
 import yaml
+import traceback
+import logging
 
 
 class MonthRevenueFetcher(StatsFetcher):
@@ -11,151 +13,183 @@ class MonthRevenueFetcher(StatsFetcher):
         iFa.ai: 財務分析 -> 每月營收
     """
 
-    def __init__(self, ticker, db_client):
+    def __init__(self, ticker, db_client, logger = None):
         super().__init__(ticker, db_client)
+        self.logger = logger or logging.getLogger(__name__)
 
-    def prepare_query(self, target_year, target_month):
+    def _prepare_query(self, target_year, target_month):
         pipeline = super().prepare_query()
 
-        pipeline.append({
-            "$project": {
-                "_id": 0,
-                "ticker": 1,
-                "company_name": 1,
-                "monthly_data": {
-                    "$sortArray": {
-                        "input": "$monthly_data",
-                        "sortBy": {
-                            "year": -1,
-                            "month": -1
+        pipeline.append(
+            {
+                "$project": {
+                    "_id": 0,
+                    "ticker": 1,
+                    "company_name": 1,
+                    "monthly_data": {
+                        "$sortArray": {
+                            "input": "$monthly_data",
+                            "sortBy": {
+                                "year": -1,
+                                "month": -1
+                            }
                         }
-                    }
-                },
+                    },
+                }
             }
-        })
+        )
 
         return pipeline
 
     def collect_data(self, target_year, target_month):
-        pipeline = self.prepare_query(target_year, target_month)
-
+        pipeline = self._prepare_query(target_year, target_month)
         fetched_data = self.collection.aggregate(pipeline)
+        fetched_data = fetched_data.to_list()
 
-        fetched_data = list(fetched_data)
-
-        return fetched_data[-1]
+        return fetched_data
 
     def query_data(self):
-        try:
-            latest_time = StatsDateTime.get_latest_time(
-                self.ticker, self.collection)['last_update_time']
-            target_year = latest_time['monthly_data']['latest_year']
-            target_month = latest_time['monthly_data']['latest_month']
-        except Exception as e:
-            today = StatsDateTime.get_today()
-            target_month = today.month
-            target_year = today.year
+        target_year, target_month = self._get_target_year_and_month()
 
         # Query data
         fetched_data = self.collect_data(target_year, target_month)
 
-        return self.process_data(fetched_data)
+        try:
+            return self._process_data(fetched_data[-1])
+        except Exception:
+            recent_date = []
+            for _ in range(12):
+                recent_date.append(f"{target_year}/{target_month}")
+                target_year, target_month = (
+                    target_year - 1, 12
+                ) if target_month == 1 else (target_year, target_month - 1)
+
+            # logging.warning(f"{self.ticker}: No monthly revenue data in TWSE mongoDB", exc_info=True)
+            return self._get_empty_structure(target_year, target_month)
+
+    def _process_data(self, fetched_data):
 
-    def process_data(self, fetched_data):
+        monthly_data = fetched_data.get('monthly_data', [])
+        if not monthly_data:
+            raise ValueError("monthly_data is empty or missing")
 
-        monthly_data = fetched_data['monthly_data']
         for data in monthly_data:
             for key, value in data.items():
-                if ("YoY" in key):
+                if "YoY" in key:
                     data[key] = StatsProcessor.cal_percentage(value)
-                elif ("ratio" in key or 'percentage' in key):
-                    data[key] = StatsProcessor.cal_non_percentage(value,
-                                                                  to_str=True,
-                                                                  postfix="%")
-                elif (key not in ('year', 'month')):
-                    data[key] = StatsProcessor.cal_non_percentage(value,
-                                                                  postfix="千元")
-        target_month = monthly_data[0]['month']
-        monthly_df = pd.DataFrame(monthly_data)
+                elif "ratio" in key or "percentage" in key:
+                    data[key] = StatsProcessor.cal_non_percentage(value, to_str=True, postfix="%")
+                elif key not in ('year', 'month'):
+                    data[key] = StatsProcessor.cal_non_percentage(value, postfix="千元")
 
+        monthly_df = pd.DataFrame(monthly_data)
+        target_month = monthly_data[0]['month']
         target_month_df = monthly_df[monthly_df['month'] == target_month]
         annual_month_df = monthly_df[monthly_df['month'] == 12]
-        month_revenue_df = monthly_df.pivot(index='month',
-                                            columns='year',
-                                            values='revenue')
-
-        grand_total_df = target_month_df.pivot(index='month',
-                                               columns='year',
-                                               values='grand_total')
 
-        annual_total_df = annual_month_df.pivot(index='month',
-                                               columns='year',
-                                               values='grand_total')
-
-        grand_total_df.rename(index={target_month: f"grand_total"},
-                              inplace=True)
+        
+        month_revenue_df = monthly_df.pivot(
+            index='month', columns='year', values='revenue'
+        )
         month_revenue_df = month_revenue_df.sort_index(ascending=False)
-        month_revenue_df = pd.concat([grand_total_df, month_revenue_df],
-                                     axis=0)
-
-        fetched_data['month_revenue'] = month_revenue_df[sorted(
-            month_revenue_df.columns, reverse=True)]
-        # 歷年月營收
-        fetched_data[
-            'this_month_revenue_over_years'] = target_month_df.set_index(
-                "year")[[
-                    "revenue", "revenue_increment_ratio", "YoY_1", "YoY_3",
-                    "YoY_5", "YoY_10"
-                ]].T
-        # 歷年營收成長量
-        fetched_data['grand_total_over_years'] = target_month_df.set_index(
-            "year")[[
-                "grand_total", "grand_total_increment_ratio",
-                "grand_total_YoY_1", "grand_total_YoY_3", "grand_total_YoY_5",
-                "grand_total_YoY_10"
-            ]].T
 
-        fetched_data.pop("monthly_data")
+        grand_total_df = target_month_df.pivot(
+            index='month', columns='year', values='grand_total'
+        )
+        grand_total_df.rename(
+            index={target_month: f"grand_total"}, inplace=True
+        )
+        month_revenue_df = pd.concat([grand_total_df, month_revenue_df], axis=0)
 
-        fetched_data['recent_month_revenue'] = self.get_recent_revenue_grwoth(
-            monthly_data, grand_total_dict=annual_total_df.to_dict(), interval = 12
+        annual_total_df = annual_month_df.pivot(
+            index='month', columns='year', values='grand_total'
+        )
+
+        fetched_data.update(
+            {
+                "month_revenue": month_revenue_df[sorted(month_revenue_df.columns, reverse=True)],
+                "this_month_revenue_over_years": target_month_df.set_index("year")[[
+                    "revenue", "revenue_increment_ratio", "YoY_1", "YoY_3", "YoY_5", "YoY_10"
+                ]].T,
+                "grand_total_over_years": target_month_df.set_index("year")[[
+                    "grand_total", "grand_total_increment_ratio", "grand_total_YoY_1", "grand_total_YoY_3", "grand_total_YoY_5", "grand_total_YoY_10"
+                ]].T,
+                "recent_month_revenue": self._get_recent_growth(monthly_data, grand_total_dict=annual_total_df.to_dict(), interval=12)
+            }
         )
 
+        fetched_data.pop("monthly_data")
         return fetched_data
 
-    def get_recent_revenue_grwoth(self, monthly_data, grand_total_dict, interval: int = 12):
-        recent_month_data = monthly_data[:interval + 1]
+    def _get_recent_growth(self, monthly_data, grand_total_dict, interval=12):
+        last_month_data = monthly_data[1:interval + 1] + [{}] * max(0, interval - len(monthly_data) + 1)
 
         MoMs = [
-            YoY_Calculator.cal_growth(this_value['revenue'], last_value['revenue'], delta = 1)
-            for this_value, last_value in zip(
-                recent_month_data[:12], recent_month_data[1:13]
-            )
+            YoY_Calculator.cal_growth(this.get('revenue'), last.get('revenue'), delta = 1)
+            for this, last in zip(monthly_data[:interval], last_month_data[:interval])
         ]
 
-        recent_month_data = {
-            "date" : [f"{data['year']}/{data['month']}" for data in recent_month_data[:interval]],
-            "revenue" : [data['revenue'] for data in recent_month_data[:interval]],
-            "MoM" : [f"{(data * 100):.2f}%" for data in MoMs],
-            "YoY" : [f"{data['revenue_increment_ratio']}" for data in recent_month_data[:interval]],
-            "total_YoY":  [f"{data['grand_total_increment_ratio']}" for data in recent_month_data[:interval]],
-        }
-
-        # accum_YoY
-        # accum_YoY 為 Davis提出的定義
-        # 2024/6的累計YoY(accum_YoY) 為 2024累計到6月為止的總營收/2023年度總營收
-        accum_YoYs = []
-        for data in monthly_data[:interval]:
+        def safe_accum_yoy(data):
             try:
                 year = data['year'] - 1
                 total = grand_total_dict[year][12]
-                accum_YoY = round(((data['grand_total'] - total) / total) * 100, 2)
-                accum_YoYs.append(f"{accum_YoY}%")
-            except Exception as e:
-                accum_YoYs.append(None)
-        
-        recent_month_data['accum_YoY'] = accum_YoYs
+                grand_total = data.get('grand_total')
+                return f"{round(((grand_total - total) / total) * 100, 2)}%"
+            except Exception:
+                self.logger.debug(f"accum_YoY calc failed for year={data.get('year')} / ticker={self.ticker}", exc_info=True)
+                return None
 
-        recent_month_df = pd.DataFrame(recent_month_data).set_index('date').T
+        recent_month_data = {
+            "date": [f"{d.get('year', 0)}/{d.get('month', 0)}" for d in monthly_data[:interval]],
+            "revenue": [d.get('revenue') for d in monthly_data[:interval]],
+            "MoM": [f"{(m * 100):.2f}%" if isinstance(m, float) else None for m in MoMs],
+            "YoY": [d.get('revenue_increment_ratio') for d in monthly_data[:interval]],
+            "total_YoY": [d.get('grand_total_increment_ratio') for d in monthly_data[:interval]],
+            # accum_YoY
+            # accum_YoY 為 Davis提出的定義
+            # 2024/6的累計YoY(accum_YoY) 為 2024累計到6月為止的總營收/2023年度總營收
+            "accum_YoY": [safe_accum_yoy(d) for d in monthly_data[:interval]]
+        }
 
-        return recent_month_df
+        df = pd.DataFrame(recent_month_data)
+        return df[df['date'] != "0/0"].set_index('date').T
+
+
+    def _get_empty_structure(self, target_year, target_month):
+        """
+        Exception 發生時回傳
+        """
+        recent_date = [f"{target_year}/{target_month}"]
+        for _ in range(11):
+            target_year, target_month = (target_year - 1, 12) if target_month == 1 else (target_year, target_month - 1)
+            recent_date.append(f"{target_year}/{target_month}")
+
+        def empty_df(index, columns):
+            return pd.DataFrame(index=index, columns=columns)
+
+        return {
+            "month_revenue": empty_df(
+                index=pd.Index(['grand_total', 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], dtype='object', name='month'),
+                columns=pd.Index([f"{target_year - i}" for i in range(10)], dtype=object, name='year')
+            ),
+            "this_month_revenue_over_years": empty_df(
+                index=pd.Index(['revenue', 'revenue_increment_ratio', 'YoY_1', 'YoY_3', 'YoY_5', 'YoY_10'], dtype='object'),
+                columns=pd.Index([f"{target_year - i}" for i in range(10)], dtype='int64', name='year')
+            ),
+            "grand_total_over_years": empty_df(
+                index=pd.Index(['grand_total', 'grand_total_increment_ratio', 'grand_total_YoY_1', 'grand_total_YoY_3', 'grand_total_YoY_5', 'grand_total_YoY_10'], dtype='object'),
+                columns=pd.Index([f"{target_year - i}" for i in range(10)], dtype='int64', name='year')
+            ),
+            "recent_month_revenue": empty_df(
+                index=pd.Index(['revenue', 'MoM', 'YoY', 'total_YoY', 'accum_YoY'], dtype='object'),
+                columns=pd.Index([], dtype = 'object', name = 'date')
+            )
+        }
+    
+    def _get_target_year_and_month(self):
+        try:
+            latest_time = StatsDateTime.get_latest_time(self.ticker, self.collection)['last_update_time']
+            return latest_time['monthly_data']['latest_year'], latest_time['monthly_data']['latest_month']
+        except Exception:
+            today = StatsDateTime.get_today()
+            return today.year, today.month