neurostats-API 0.0.25rc1__py3-none-any.whl → 1.0.0rc2__py3-none-any.whl

neurostats_API/async_mode/fetchers/cash_flow.py

@@ -0,0 +1,56 @@
+from .base import AsyncBaseFetcher
+from ..db_extractors import AsyncCashFlowExtractor
+from neurostats_API.async_mode.factory import get_extractor
+from neurostats_API.transformers import (
+    TWSECashFlowTransformer, USCashFlowTransformer
+)
+from neurostats_API.utils import NoCompanyError, NotSupportedError
+
+
+class AsyncCashFlowFetcher(AsyncBaseFetcher):
+
+    def __init__(self, ticker, client):
+
+        super().__init__()
+        self.transformer_map = {
+            "tw": TWSECashFlowTransformer,
+            "us": USCashFlowTransformer
+        }
+        self.ticker = ticker
+        try:
+            self.extractor = get_extractor("DB_cash_flow", self.ticker, client)
+            self.company_name = self.extractor.get_company_name()
+            self.zone = self.extractor.get_zone()
+
+            transformer = self.get_transformer(self.zone)
+            self.transformer = transformer(ticker, self.company_name, self.zone)
+        
+        except NotSupportedError as e:
+            raise NotSupportedError(
+                f"{self.__class__.__name__} only support {list(self.transformer_map.keys())} companies now, {ticker} is not available"
+            ) from e
+        except NoCompanyError:
+            raise
+
+    async def query_data(self, start_date=None, end_date=None):
+        fetched_data = await self.extractor.query_data(start_date, end_date)
+
+        transformed_data = self.transformer.process_transform(fetched_data)
+
+        return transformed_data
+
+    async def query_data_stats_only(self, start_date=None, end_date=None):
+        fetched_data = await self.extractor.query_data(start_date, end_date)
+
+        transformed_data = self.transformer.process_stats_page(fetched_data)
+
+        return transformed_data
+
+    async def query_data_QoQ_and_YoY_only(self, start_date=None, end_date=None):
+        if (self.zone != 'tw'):
+            raise NotSupportedError("query_data_QoQ_and_YoY_only() only supports for TW companies")
+        fetched_data = await self.extractor.query_data(start_date, end_date)
+
+        transformed_data = self.transformer.process_QoQ(fetched_data)
+
+        return transformed_data

neurostats_API/async_mode/fetchers/finance_overview.py

@@ -0,0 +1,134 @@
+from .base import AsyncBaseFetcher
+from datetime import datetime
+import importlib.resources as pkg_resources
+
+from neurostats_API.async_mode.db_extractors import (
+    AsyncBalanceSheetExtractor, AsyncCashFlowExtractor, AsyncProfitLoseExtractor
+)
+from neurostats_API.async_mode.factory import get_extractor
+from neurostats_API.transformers import AgentOverviewTransformer, FinanceOverviewTransformer
+from neurostats_API.utils import StatsDateTime, StatsProcessor, NoCompanyError, NotSupportedError
+import numpy as np
+import pandas as pd
+import pytz
+import holidays
+import warnings
+
+warnings.filterwarnings(
+    "ignore",
+    category=holidays.deprecations.v1_incompatibility.
+    FutureIncompatibilityWarning
+)
+import yaml
+
+
+class AsyncFinanceOverviewFetcher(AsyncBaseFetcher):
+    """
+    對應iFa.ai -> 財務分析 -> 重要指標(finance_overview)
+    """
+
+    def __init__(self, ticker, db_client):
+        super().__init__()
+
+        self.target_fields = StatsProcessor.load_yaml(
+            "twse/finance_overview_dict.yaml"
+        )
+        self.inverse_dict = StatsProcessor.load_txt(
+            "twse/seasonal_data_field_dict.txt", json_load=True
+        )
+        try:
+            self.balance_sheet_extractor = get_extractor(
+                "DB_balance_sheet", ticker, db_client
+            )
+            self.profit_lose_extractor = get_extractor(
+                "DB_profit_lose", ticker, db_client
+            )
+            self.cashflow_extractor = get_extractor(
+                "DB_cash_flow", ticker, db_client
+            )
+
+            company_name = self.balance_sheet_extractor.get_company_name()
+            zone = self.balance_sheet_extractor.get_zone()
+
+            if (zone != 'tw'):
+                raise NotSupportedError(None)
+
+            self.transformer = FinanceOverviewTransformer(
+                ticker, company_name, zone
+            )
+        except NotSupportedError as e:
+            raise NotSupportedError("FinanceOverviewFetcher only supports TW corporation now") from e
+
+    async def query_data(self, date=None):
+
+        balance_sheet = await self.balance_sheet_extractor.query_data(
+            end_date=date, get_latest=True
+        )
+        profit_lose = await self.profit_lose_extractor.query_data(
+            end_date=date, get_latest=True
+        )
+        cash_flow = await self.cashflow_extractor.query_data(
+            end_date=date, get_latest=True
+        )
+
+        transformed_data = self.transformer.process_transform(
+            balance_sheet, profit_lose, cash_flow
+        )
+
+        return transformed_data
+
+
+class AsyncAgentOverviewFetcher(AsyncBaseFetcher):
+    """
+    用於P1-Agent 的 公司股市經濟概況頁面
+    """
+
+    def __init__(self, ticker, db_client):
+        from neurostats_API.async_mode.fetchers import AsyncTechFetcher
+        self.ticker = ticker
+        self.tech_fetcher = AsyncTechFetcher(
+            ticker=self.ticker, client=db_client
+        )
+        try:
+            self.balance_sheet_extractor = AsyncBalanceSheetExtractor(
+                ticker=self.ticker, client=db_client
+            )
+            self.profit_lose_extractor = AsyncProfitLoseExtractor(
+                ticker=self.ticker, client=db_client
+            )
+            self.cash_flow_extractor = AsyncCashFlowExtractor(
+                ticker=self.ticker, client=db_client
+            )
+
+            company_name = self.balance_sheet_extractor.get_company_name()
+            zone = self.balance_sheet_extractor.get_zone()
+
+            self.transformer_map = {'us': AgentOverviewTransformer}
+
+            transformer = self.get_transformer(zone)
+            self.transformer = transformer(ticker, company_name, zone)
+
+        except NotSupportedError as e:
+            raise NotSupportedError(
+                f'Agent Overview only supports US corporation now, Got zone={zone}, ticker={ticker}'
+            ) from e
+
+    async def query_data(self, date=None):
+
+        tech_data = await self.tech_fetcher.get_daily(
+            start_date=None, end_date=date
+        )
+        balance_sheet = await self.balance_sheet_extractor.query_data(
+            end_date=date
+        )
+        profit_lose = await self.profit_lose_extractor.query_data(end_date=date)
+        cash_flow = await self.cash_flow_extractor.query_data(end_date=date)
+
+        return_data = self.transformer.process_transform(
+            tech_data=tech_data,
+            balance_sheet_list=balance_sheet,
+            profit_lose_list=profit_lose,
+            cash_flow_list=cash_flow
+        )
+
+        return return_data

neurostats_API/async_mode/fetchers/month_revenue.py

@@ -0,0 +1,35 @@
+from .base import AsyncBaseFetcher
+from ..db_extractors import AsyncTWSEMonthlyRevenueExtractor
+from neurostats_API.async_mode.factory import get_extractor
+from neurostats_API.transformers import TWSEMonthlyRevenueTransformer
+from neurostats_API.utils import NoCompanyError, NotSupportedError
+
+
+class AsyncMonthlyRevenueFetcher(AsyncBaseFetcher):
+
+    def __init__(self, ticker, client):
+        self.ticker = ticker
+        self.transformer_map = {
+            "tw": TWSEMonthlyRevenueTransformer,
+        }
+
+        try:
+            self.extractor = get_extractor("TWSE_month_revenue", ticker, client)
+            company_name = self.extractor.get_company_name()
+            zone = self.extractor.get_zone()
+
+            transformer = self.get_transformer(zone)
+
+            self.transformer = transformer(ticker, company_name, zone)
+        except NotSupportedError as e:
+            raise NotSupportedError(
+                "AsyncMonthRevenueFetcher only supports tw company now"
+            ) from e
+
+
+    async def query_data(self, start_date=None, end_date=None):
+        fetched_data = await self.extractor.query_data(start_date, end_date)
+
+        transformed_data = self.transformer.process_transform(fetched_data)
+
+        return transformed_data

neurostats_API/async_mode/fetchers/profit_lose.py

@@ -0,0 +1,46 @@
+from neurostats_API.async_mode.db_extractors import AsyncProfitLoseExtractor
+from neurostats_API.async_mode.factory import get_extractor
+from neurostats_API.transformers import (
+    TWSEProfitLoseTransformer,
+    USProfitLoseTransformer
+)
+from neurostats_API.utils import (
+    NoCompanyError,
+    NotSupportedError
+)
+
+from .base import AsyncBaseFetcher
+
+class AsyncProfitLoseFetcher(AsyncBaseFetcher):
+    def __init__(
+        self, 
+        ticker,
+        client
+    ):
+        self.ticker = ticker
+        
+        self.transformer_map = {
+            "tw": TWSEProfitLoseTransformer,
+            "us": USProfitLoseTransformer
+        }
+        try:
+            self.extractor = get_extractor("DB_profit_lose", ticker, client)
+            company_name = self.extractor.get_company_name()
+            zone = self.extractor.get_zone()
+
+            transformer = self.get_transformer(zone)
+            self.transformer = transformer(ticker, company_name, zone)
+        
+        except NotSupportedError as e:
+            raise NotSupportedError(
+                 f"{self.__class__.__name__} only support {list(self.transformer_map.keys())} companies now, {ticker} is not available"
+            ) from e
+
+    async def query_data(self, start_date = None, end_date = None):
+        fetched_data = await self.extractor.query_data(start_date, end_date)
+
+        transformed_data = self.transformer.process_transform(
+            fetched_data
+        )
+
+        return transformed_data

neurostats_API/async_mode/fetchers/tech.py

@@ -0,0 +1,205 @@
+from .base import AsyncBaseFetcher
+from datetime import datetime
+from ..db_extractors import (
+    AsyncYFDailyTechDBExtractor, AsyncTEJDailyTechDBExtractor
+)
+from neurostats_API.async_mode.factory import get_extractor
+from neurostats_API.transformers import DailyTechTransformer
+from neurostats_API.utils import NotSupportedError
+import pandas as pd
+import yfinance as yf
+
+
+class AsyncTechFetcher(AsyncBaseFetcher):
+
+    def __init__(self, ticker, client):
+        self.ticker = ticker
+        self.transformer_map = {
+            "tw": DailyTechTransformer,
+            "us": DailyTechTransformer
+        }
+        try:
+            self.yf_db_extractor = get_extractor("YF_tech", ticker, client)
+
+            self.tej_db_extractor = None
+            if (self.yf_db_extractor.get_zone() == 'tw'):
+                self.tej_db_extractor = get_extractor("TEJ_tech", ticker, client)
+
+            self.company_name = self.yf_db_extractor.get_company_name()
+            self.zone = self.yf_db_extractor.get_zone()
+
+            self.required_cols = [
+                'date', 'open', 'high', 'low', 'close', 'volume'
+            ]
+
+            transformer = self.get_transformer(self.zone)
+            self.transformer = transformer(ticker, self.company_name, self.zone)
+
+        except NotSupportedError as e:
+            raise NotSupportedError(
+                f"{self.__class__.__name__} only supports {list(self.transformer_map.keys())}, got {self.zone} with ticker = \"{self.ticker}\""
+            ) from e
+
+    async def query_data(self, start_date=None, end_date=None):
+        if (start_date is None):
+            start_date = datetime.strptime("1991-01-01", "%Y-%m-%d")
+        if (end_date is None):
+            end_date = datetime.today()
+
+        fetched_data = await self._fetch_data(start_date, end_date)
+
+        transformed_data = self.transformer.process_transform(fetched_data)
+
+        return transformed_data
+
+    async def get_daily(self, start_date=None, end_date=None):
+        await self.query_data(start_date, end_date)
+
+        return self.transformer.get_daily()
+
+    async def get_weekly(self, start_date=None, end_date=None):
+        await self.query_data(start_date, end_date)
+
+        return self.transformer.get_weekly()
+
+    async def get_monthly(self, start_date=None, end_date=None):
+        await self.query_data(start_date, end_date)
+
+        return self.transformer.get_monthly()
+
+    async def get_quarterly(self, start_date=None, end_date=None):
+        await self.query_data(start_date, end_date)
+
+        return self.transformer.get_quarterly()
+
+    async def get_yearly(self, start_date=None, end_date=None):
+        await self.query_data(start_date, end_date)
+
+        return self.transformer.get_yearly()
+
+    async def _fetch_data(self, start_date, end_date):
+
+        if (self.ticker in ['GSPC', 'IXIC', 'DJI', 'TWII']):
+            full_tick = f'^{self.ticker}'
+            df = self._conduct_yf_search(full_tick, start_date, end_date)
+
+            return df[self.required_cols]
+
+        else:
+            search_fn_map = {
+                'tw': self._conduct_tw_db_search,
+                'us': self._conduct_us_db_search
+            }
+            search_fn = search_fn_map.get(self.zone)
+            df = await search_fn(start_date, end_date)
+
+            return df
+
+    def _conduct_yf_search(self, ticker, start_date, end_date):
+        yf_ticker = yf.Ticker(ticker)
+        origin_df = yf_ticker.history(period="10y")
+
+        if origin_df.empty:
+            return origin_df
+
+        origin_df = origin_df.reset_index()
+        origin_df["Date"] = pd.to_datetime(origin_df["Date"])
+        df = origin_df.rename(
+            columns={
+                "Date": "date",
+                "Open": "open",
+                "High": "high",
+                "Low": "low",
+                "Close": "close",
+                "Volume": "volume"
+            }
+        )
+
+        df = df.loc[(df['date'] >= start_date) & (df['date'] <= end_date)]
+        return df
+
+    async def _conduct_tw_db_search(self, start_date, end_date):
+        search_fns = [
+            self._conduct_tw_db_search_yf,
+            self._conduct_tw_db_search_tej,
+        ]
+
+        sync_search_fns = [
+            lambda s_date, e_date: self.
+            _conduct_yf_search(f'{self.ticker}.tw', s_date, e_date),
+            lambda s_date, e_date: self.
+            _conduct_yf_search(f'{self.ticker}.two', s_date, e_date),
+        ]
+
+        for search_method in search_fns:
+            try:
+                fetched_data = await search_method(start_date, end_date)
+                if not fetched_data.empty:
+                    break
+            except (KeyError, ValueError, TypeError) as e:
+                print(e)
+                continue
+        else:    # 找不到資料
+            for search_method in sync_search_fns:
+                fetched_data = search_method(start_date, end_date)
+                if not fetched_data.empty:
+                    break
+            else:
+                return pd.DataFrame(self.required_cols)
+
+        return fetched_data[self.required_cols]
+
+    async def _conduct_tw_db_search_yf(self, start_date, end_date):
+        """
+        尋找從yf與twse爬下來的台股資料
+        """
+        fetched_data = await self.yf_db_extractor.query_data(
+            start_date, end_date
+        )
+
+        if (len(fetched_data) > 0):
+            df = pd.DataFrame(fetched_data)
+            return df
+        else:
+            return None
+
+    async def _conduct_tw_db_search_tej(self, start_date, end_date):
+        """
+        尋找從TEJ爬下來的台股資料
+        """
+        fetched_data = await self.tej_db_extractor.query_data(
+            start_date, end_date
+        )
+
+        if (len(fetched_data) > 0):
+            df = pd.DataFrame(fetched_data)
+            df = df.rename(
+                columns={
+                    "open_d": "open",
+                    "high_d": "high",
+                    "low_d": "low",
+                    "close_d": "close",
+                    "vol": "volume"
+                }
+            )
+
+            return df
+        else:
+            return None
+
+    def _conduct_us_db_search(self, start_date, end_date):
+        search_fns = [
+            self.yf_db_extractor.query_data, lambda s_date, e_date: self.
+            _conduct_yf_search(f"{self.ticker}", s_date, e_date)
+        ]
+
+        for search_method in search_fns:
+            try:
+                df = search_method(start_date, end_date)
+                break
+            except (KeyError, ValueError, TypeError):
+                continue
+        else:    # 回傳空df
+            df = pd.DataFrame(columns=self.required_cols)
+
+        return df

neurostats_API/async_mode/fetchers/tej.py

@@ -0,0 +1,88 @@
+from .base import AsyncBaseFetcher
+from datetime import datetime
+from neurostats_API.async_mode.db_extractors import (
+    AsyncTEJFinanceStatementDBExtractor, AsyncTEJSelfSettlementDBExtractor
+)
+from neurostats_API.async_mode.factory import get_extractor
+from neurostats_API.transformers import TEJFinanceStatementTransformer
+from neurostats_API.utils import NotSupportedError
+
+
+class AsyncTEJSeasonalFetcher(AsyncBaseFetcher):
+
+    def __init__(
+        self,
+        ticker,
+        client,
+        collection='TEJ_finance_statement',
+        fetch_type='Q'
+    ):
+        self.ticker = ticker
+        self.fetch_type = fetch_type
+        self.transformer_map = {"tw": TEJFinanceStatementTransformer}
+        try:
+            self.db_extractor = get_extractor(
+                collection=collection,
+                ticker=ticker,
+                client=client,
+                fetch_type=fetch_type
+            )
+
+            self.company_name = self.db_extractor.get_company_name()
+            self.zone = self.db_extractor.get_zone()
+
+            
+
+            transformer = self.get_transformer(self.zone)
+
+            self.transformer = transformer(
+                ticker=ticker,
+                company_name=self.company_name,
+                zone=self.zone,
+                target_column=fetch_type
+            )
+
+        except NotSupportedError as e:
+            raise NotSupportedError(
+                f"{self.__class__.__name__} only support {list(self.transformer_map.keys())} companies now, {ticker} is not available"
+            ) from e
+
+    async def query_data(
+        self,
+        start_date=None,
+        end_date=None,
+        fetch_mode='QoQ',
+        use_cal=True,
+        indexes=None
+    ):
+        fetched_data = await self.db_extractor.query_data(
+            start_date=None, end_date=end_date
+        )    # 先索取目標日期之前的之後再截斷
+
+        if (start_date is None):
+            start_date = datetime.strptime("1991-01-01", "%Y-%m-%d")
+
+        transformed_data = self.transformer.process_transform(
+            fetched_data=fetched_data,
+            fetch_mode=fetch_mode,
+            start_date=start_date,
+            use_cal=use_cal,
+            indexes=indexes
+        )    # 轉換並根據開始日期截斷
+
+        return transformed_data
+
+    def _get_extractor(self, collection):
+        EXTRACTOR_MAP = {
+            "TEJ_finance_statement": AsyncTEJFinanceStatementDBExtractor,
+            "TEJ_self_settlement": AsyncTEJSelfSettlementDBExtractor,
+        }
+
+        extractor = EXTRACTOR_MAP.get(collection, None)
+
+        if (extractor is None):
+            raise ValueError(
+                f"{collection} not a regular argument for collection, only {list(EXTRACTOR_MAP.keys())} available"
+            )
+
+        return extractor

neurostats_API/async_mode/fetchers/twse_institution.py

@@ -0,0 +1,62 @@
+from .base import AsyncBaseFetcher
+from datetime import datetime, timedelta
+from ..db_extractors import (
+    AsyncTWSEChipDBExtractor,
+    AsyncYFDailyTechDBExtractor,
+)
+from neurostats_API.async_mode.factory import get_extractor
+from neurostats_API.utils import NotSupportedError
+from neurostats_API.transformers import TWSEChipTransformer
+
+
+class AsyncTWSEInstitutionFetcher(AsyncBaseFetcher):
+
+    def __init__(self, ticker, client):
+        self.ticker = ticker
+        self.transformer_map = {"tw": TWSEChipTransformer}
+        try:
+            self.extractor = get_extractor(
+                "TWSE_chip", ticker, client, fetch_type='I'
+            )
+            self.daily_extractor = get_extractor("YF_tech", ticker, client)
+
+            company_name = self.extractor.get_company_name()
+            zone = self.extractor.get_zone()
+
+            transformer = self.get_transformer(zone)
+            self.transformer = transformer(ticker, company_name, zone)
+
+        except NotSupportedError as e:
+            raise NotSupportedError(
+                f"{self.__class__.__name__} only support {list(self.transformer_map.keys())} companies now, {ticker} is not available"
+            ) from e
+
+    async def query_data(
+        self, start_date=None, end_date=None, get_latest=False
+    ):
+        fetched_data = await self.extractor.query_data(
+            start_date, end_date, get_latest
+        )
+        tech_data = await self.daily_extractor.query_data(
+            start_date, end_date, get_latest
+        )
+
+        fetched_data = {"institution_trading": fetched_data}
+        transformed_data = self.transformer.process_transform(
+            tech_data=tech_data, **fetched_data
+        )
+
+        return transformed_data
+
+    async def query_data_annual(self):
+        """
+        取一年內的資料
+        """
+        end_date = datetime.today()
+        start_date = end_date - timedelta(days=365)
+
+        fetched_data = await self.query_data(
+            start_date, end_date, get_latest=False
+        )
+
+        return fetched_data