neurostats-API 0.0.21b0__tar.gz → 0.0.23__tar.gz

neurostats_api-0.0.23/MANIFEST.in

@@ -0,0 +1,9 @@
+recursive-include neurostats_api/tools/tej_db *.yaml
+recursive-include neurostats_api/tools/tej_db *.txt
+recursive-include neurostats_api/tools/tej_db *.json
+recursive-include neurostats_api/tools/twse *.yaml
+recursive-include neurostats_api/tools/twse *.txt
+recursive-include neurostats_api/tools/twse *.json
+recursive-include neurostats_api/tools/company_list *.yaml
+recursive-include neurostats_api/tools/company_list *.txt
+recursive-include neurostats_api/tools/company_list *.json

{neurostats_api-0.0.21b0 → neurostats_api-0.0.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: neurostats_API
-Version: 0.0.21b0
+Version: 0.0.23
 Summary: The service of NeuroStats website
 Home-page: https://github.com/NeurowattStats/NeuroStats_API.git
 Author: JasonWang@Neurowatt
@@ -89,7 +89,7 @@ pip install neurostats-API
 ```Python 
 >>> import neurostats_API
 >>> print(neurostats_API.__version__)
-0.0.21b0
+0.0.23
 ```
 
 ### 得到最新一期的評價資料與歷年評價

{neurostats_api-0.0.21b0 → neurostats_api-0.0.23}/README.md

@@ -73,7 +73,7 @@ pip install neurostats-API
 ```Python 
 >>> import neurostats_API
 >>> print(neurostats_API.__version__)
-0.0.21b0
+0.0.23
 ```
 
 ### 得到最新一期的評價資料與歷年評價

{neurostats_api-0.0.21b0 → neurostats_api-0.0.23}/neurostats_API/__init__.py

@@ -1,4 +1,4 @@
-__version__='0.0.20'
+__version__='0.0.23'
 
 from .fetchers import (
     BalanceSheetFetcher,

neurostats_api-0.0.23/neurostats_API/fetchers/balance_sheet.py

@@ -0,0 +1,201 @@
+from .base import StatsFetcher, StatsDateTime
+import json
+import pandas as pd
+from ..utils import StatsDateTime, StatsProcessor
+import yaml
+
+
+class BalanceSheetFetcher(StatsFetcher):
+    """
+    對應iFa.ai -> 財務分析 ->  資產負債表
+    """
+
+    def __init__(self, ticker, db_client):
+        super().__init__(ticker, db_client)
+        self.table_settings = StatsProcessor.load_yaml("twse/balance_sheet.yaml")
+
+        self.process_function_map = {
+            "twse_stats": self.process_data_twse,
+            "us_stats": self.process_data_us
+        }
+
+        self.return_keys = [
+            'balance_sheet', 'total_asset', 'current_asset', 'non_current_asset', 
+            'current_debt', 'non_current_debt', 'equity', 'balance_sheet_all', 'balance_sheet_YoY'
+        ]
+
+    def prepare_query(self):
+        pipeline = super().prepare_query()
+
+        name_map = {
+            "twse_stats": "balance_sheet",
+            "us_stats": "balance_sheet"
+        }
+
+
+        chart_name = name_map.get(self.collection_name, "balance_sheet")
+
+        append_pipeline = [
+        {
+            "$project": {
+                "_id": 0,
+                "ticker": 1,
+                "company_name": 1,
+                "seasonal_data": {
+                    "$map": {
+                        "input": {"$ifNull": ["$seasonal_data", []]},
+                        "as": "season",
+                        "in": {
+                            "year": "$$season.year",
+                            "season": "$$season.season",
+                            "data": {"$ifNull": [f"$$season.{chart_name}", []]}
+                        }
+                    }
+                }
+            }
+        }
+    ]
+        pipeline = pipeline + append_pipeline
+
+        return pipeline
+
+    def collect_data(self):
+        return super().collect_data()
+
+    def query_data(self):
+        fetched_data = self.collect_data()
+        fetched_data = fetched_data[0]
+
+        process_fn = self.process_function_map[self.collection_name]
+        processed_data = process_fn(fetched_data)
+        return processed_data
+
+    def process_data_twse(self, fetched_data):
+        latest_time = StatsDateTime.get_latest_time(
+            self.ticker, self.collection
+        ).get('last_update_time', {})
+        # 取最新時間資料時間，沒取到就預設去年年底
+        target_year = latest_time.get('seasonal_data', {}).get(
+            'latest_target_year',
+            StatsDateTime.get_today().year - 1
+        )
+        target_season = latest_time.get('seasonal_data',
+                                        {}).get('latest_season', 4)
+
+        return_dict = {
+            "ticker": self.ticker,
+            "company_name": fetched_data['company_name']
+        }
+        table_dict = {}
+
+        seasonal_data = fetched_data.get('seasonal_data')
+        if not seasonal_data:
+            return_dict.update(self._get_empty_structure())
+            return return_dict
+
+        # 將value與percentage跟著年分季度一筆筆取出
+        for data in seasonal_data:
+            year, season, balance_sheet = data['year'], data['season'], data[
+                'data']
+            time_index = f"{year}Q{season}"
+
+            new_balance_sheet = dict()
+            # 蒐集整體的keys
+            index_names = list(balance_sheet.keys())
+
+            table_dict[time_index] = balance_sheet
+            # flatten dict:
+            # {<key>: {"value": <value>, "percentage": <value>}}
+            # -> {<key>_value: <value>, <key>_percentage:<value>}
+        
+        old_balance_sheet = pd.DataFrame(table_dict)
+        target_season_col = old_balance_sheet.columns.str.endswith(f"Q{target_season}")
+        old_balance_sheet = old_balance_sheet.loc[:, target_season_col]
+        old_balance_sheet = StatsProcessor.expand_value_percentage(old_balance_sheet)
+
+        # 處理QoQ版BalanceSheet
+        for time_index, data_dict in table_dict.items():
+            new_balance_sheet = self.flatten_dict(
+                data_dict, 
+                indexes = index_names,
+                target_keys=["value", "percentage"]
+            )
+
+            table_dict[time_index] = new_balance_sheet
+
+        total_table = pd.DataFrame.from_dict(table_dict).T
+        value_index = total_table.columns.str.endswith("_value")
+        value_cols = total_table.loc[:, value_index].columns
+        total_table[value_cols] = (
+            total_table[value_cols].map(
+                lambda x: StatsProcessor.cal_non_percentage(x, postfix="千元"),
+            )
+        )
+
+        percentage_index = total_table.columns.str.endswith(
+            "_percentage"
+        )
+        percentage_cols = total_table.loc[:, percentage_index].columns
+        total_table[percentage_cols] = (
+            total_table[percentage_cols].map(
+                lambda x: StatsProcessor.
+                cal_non_percentage(x, to_str=True, postfix="%"),
+            )
+        )
+
+        total_table = total_table.T
+        target_season_columns = total_table.columns.str.endswith(
+            f"Q{target_season}"
+        )
+        total_table_YoY = total_table.loc[:, target_season_columns]
+
+        for name, setting in self.table_settings.items():
+            target_indexes = setting.get('target_index', [None])
+            for target_index in target_indexes:
+                try:
+                    return_dict[name] = StatsProcessor.slice_old_table(
+                        total_table=old_balance_sheet,
+                        target_index=target_index
+                    )
+                    break
+                except Exception as e:
+                    continue
+        
+        return_dict.update(
+            {
+                "balance_sheet": old_balance_sheet,
+                "balance_sheet_all": total_table.copy(),
+                "balance_sheet_YoY": total_table_YoY
+            }
+        )
+        return return_dict
+
+    def process_data_us(self, fetched_data):
+        return_dict = {
+            "ticker": self.ticker,
+            "company_name": fetched_data[-1]['company_name']
+        }
+
+        table_dict = dict()
+
+        for data in fetched_data:
+            year, season, balance_sheet = data['year'], data['season'], data[
+                'balance_sheet']
+            table_dict[f"{year}Q{season}"] = balance_sheet
+
+        table_dict = pd.DataFrame.from_dict(table_dict)
+
+        return_dict["balance_sheet"] = table_dict
+
+        latest_season = fetched_data[0]['season']
+        target_season_columns = table_dict.columns.str.endswith(
+            f"Q{latest_season}"
+        )
+        table_dict_YoY = table_dict.loc[:, target_season_columns]
+        return_dict["balance_sheet_YoY"] = table_dict_YoY
+        return return_dict
+
+    def _get_empty_structure(self):
+        return {
+            key: pd.DataFrame(columns= pd.Index([], name = 'date')) for key in self.return_keys
+        }

{neurostats_api-0.0.21b0 → neurostats_api-0.0.23}/neurostats_API/fetchers/base.py

@@ -1,22 +1,31 @@
 import abc
-from typing import Union
-from pymongo import MongoClient
-import pandas as pd
+from datetime import datetime, timedelta, date
 import json
+import pandas as pd
+from pymongo import MongoClient
 import pytz
-from datetime import datetime, timedelta, date
+from typing import Union
 from ..utils import StatsDateTime, StatsProcessor, YoY_Calculator
-import yaml
 
 
-class StatsFetcher:
+class StatsFetcher(abc.ABC):
 
-    def __init__(self, ticker, db_client):
+    def __init__(self, ticker: str, db_client: MongoClient):
         self.ticker = ticker
-        self.db = db_client["company"]    # Replace with your database name
-        self.collection = self.db["twse_stats"]
-
         self.timezone = pytz.timezone("Asia/Taipei")
+        self.tw_company_list = StatsProcessor.load_json("company_list/tw.json")
+        db_mapping = {
+            "company": "twse_stats",
+            "company_us": "us_stats",
+        }
+
+        name_mapping = {"company": "台股", "company_us": "美股"}
+
+        db_name = "company" if self.ticker in self.tw_company_list else "company_us"
+        self.db = db_client[db_name]
+        self.collection_name = db_mapping.get(db_name, "unknown")
+        assert self.collection_name != "unknown", f"請確認 {ticker} 是否是 {','.join(list(name_mapping.values()))}"
+        self.collection = db_client[db_name][self.collection_name]
 
         self.target_metric_dict = {
             'value': ['value'],
@@ -37,40 +46,41 @@ class StatsFetcher:
                 }
             },
         ]
+    
+    def query_data(self):
+        return NotImplementedError()
 
-    def collect_data(self, start_date, end_date):
+    def collect_data(self):
         pipeline = self.prepare_query()
-
         fetched_data = list(self.collection.aggregate(pipeline))
+        return fetched_data if fetched_data else None
 
-        return fetched_data[0]
-
-    def str_to_datetime(self, date_str):
-        year, month, day = [int(num) for num in date_str.split("-")]
-
-        date = datetime.strptime(date_str, "%Y-%m-%d")
-        date = self.timezone.localize(date)
-
+    def str_to_datetime(self, date_str: str) -> StatsDateTime:
+        date = self.timezone.localize(datetime.strptime(date_str, "%Y-%m-%d"))
+        year, month, day = date.year, date.month, date.day
         season = (month - 1) // 3 + 1
-
         return StatsDateTime(date, year, month, day, season)
 
-    def has_required_columns(self, df: pd.DataFrame, required_cols=None):
-        """
-        Check if the required columns are present in the DataFrame.
-
-        Args:
-            df (pd.DataFrame): The DataFrame to check.
-            required_cols (list, optional): List of required column names. 
-                                            Defaults to ['date', 'open', 'high', 'low', 'close', 'volume'].
-
-        Returns:
-            bool: True if all required columns are present, False otherwise.
-        """
+    def has_required_columns(
+        self, df: pd.DataFrame, required_cols=None
+    ) -> bool:
         if required_cols is None:
             required_cols = ['date', 'open', 'high', 'low', 'close', 'volume']
-
         return all(col in df.columns for col in required_cols)
+    
+    @staticmethod
+    def flatten_dict(value_dict, indexes, target_keys):
+        indexes = value_dict.keys()
+        new_dict = {}
+
+        for key in indexes:
+            new_dict.update(
+                {
+                    f"{key}_{sub_key}": value_dict[key].get(sub_key, None) 
+                    for sub_key in target_keys
+                }
+            )
+        return new_dict
 
 
 class BaseTEJFetcher(abc.ABC):
@@ -81,21 +91,18 @@ class BaseTEJFetcher(abc.ABC):
 
     def get_latest_data_time(self, ticker):
         latest_data = self.collection.find_one(
-            {
-                "ticker": ticker
-            }, 
-            {
+            {"ticker": ticker}, {
                 "last_update": 1,
                 "_id": 0
             }
         )
 
-        try:
-            latest_date = latest_data['last_update']["latest_data_date"]
-        except Exception as e:
-            latest_date = None
-
-        return latest_date
+        if (latest_data):
+            # return 得到最新日期或None
+            return latest_data.get('last_update', {}).get("latest_data_date", None)
+        
+        else:
+            return None
 
     def process_value(self, value):
         if isinstance(value, str) and "%" in value:
@@ -107,77 +114,89 @@ class BaseTEJFetcher(abc.ABC):
 
     def calculate_growth(self, this_value, last_value, delta):
         try:
-            return YoY_Calculator.cal_growth(this_value, last_value, delta) * 100
+            return YoY_Calculator.cal_growth(
+                this_value, last_value, delta
+            ) * 100
         except Exception:
             return None
 
-    def cal_YoY(self, data_dict: dict, start_year: int, end_year: int, season: int):
+    def cal_YoY(
+        self, data_dict: dict, start_year: int, end_year: int, season: int
+    ):
         year_shifts = [1, 3, 5, 10]
         return_dict = {}
-        
+
         for year in range(start_year, end_year + 1):
             year_data = data_dict.get(f"{year}Q{season}", {}).copy()
             if not year_data:
                 continue
-            
-            for key in list(year_data.keys()):
+
+            for key, value in list(year_data.items()):
                 if key == "season":
                     continue
-                
-                this_value = self.process_value(year_data[key])
+
+                this_value = self.process_value(value)
                 if this_value is None:
                     year_data.pop(key)
                     continue
-                
-                temp_dict = {"value": year_data[key]}
+
+                temp_dict = {"value": value}
                 for shift in year_shifts:
-                    past_year = year - shift
-                    last_value = data_dict.get(f"{past_year}Q{season}", {}).get(key)
-                    last_value = self.process_value(last_value)
-                    growth = self.calculate_growth(this_value, last_value, shift) if last_value is not None else None
+                    past_value = self.process_value(
+                        data_dict.get(f"{year - shift}Q{season}", {}).get(key)
+                    )
+                    growth = self.calculate_growth(this_value, past_value, shift) if past_value else None
+
+                    temp_dict[
+                        f"YoY_{shift}"
+                    ] = f"{growth:.2f}%" if growth else None
 
-                    temp_dict[f"YoY_{shift}"] = (f"{growth:.2f}%" if growth else None)
                 year_data[key] = temp_dict
-            
             return_dict[f"{year}Q{season}"] = year_data
-        
         return return_dict
 
     def cal_QoQ(self, data_dict):
         return_dict = {}
-        
+
         for time_index, this_data in data_dict.items():
             year, season = map(int, time_index.split("Q"))
-            last_year, last_season = (year - 1, 4) if season == 1 else (year, season - 1)
-            
+            last_year, last_season = (
+                year - 1, 4
+            ) if season == 1 else (year, season - 1)
+
             for key in list(this_data.keys()):
                 if key == "season":
                     continue
-                
+
                 this_value = self.process_value(this_data[key])
                 if this_value is None:
                     this_data.pop(key)
                     continue
-                
+
                 temp_dict = {"value": this_data[key]}
-                last_value = data_dict.get(f"{last_year}Q{last_season}", {}).get(key, {}).get('value')
+                last_value = data_dict.get(
+                    f"{last_year}Q{last_season}",{}
+                ).get(key, {}).get('value')
+
                 last_value = self.process_value(last_value)
-                growth = self.calculate_growth(this_value, last_value, 1) if last_value is not None else None
+                growth = self.calculate_growth(
+                    this_value, last_value, 1
+                ) if last_value is not None else None
                 temp_dict['growth'] = (f"{growth:.2f}%" if growth else None)
-                
+
                 this_data[key] = temp_dict
-            
+
             return_dict[time_index] = this_data
-        
+
         return return_dict
 
     def get_dict_of_df(self, data_dict):
         """
         dict[dict] -> dict[df]
         """
-        for key in data_dict.keys():
-            data_dict[key] = pd.DataFrame.from_dict(data_dict[key])
-        return data_dict
+        return {
+            key: pd.DataFrame.from_dict(data) for key, data in data_dict.items()
+        }
 
     def set_time_shift(self, date: Union[str, datetime], period: str):
         if isinstance(date, str):
@@ -197,4 +216,4 @@ class BaseTEJFetcher(abc.ABC):
         if period == "all":
             return datetime.strptime("1991-01-01", "%Y-%m-%d")
 
-        return date - period_mapping.get(period, timedelta(days=0))  # 預設為不變"
+        return date - period_mapping.get(period, timedelta(days=0))    # 預設為不變"