neurostats-API 0.0.25rc1__tar.gz → 1.0.0rc2__tar.gz

neurostats_api-1.0.0rc2/MANIFEST.in

@@ -0,0 +1,9 @@
+recursive-include neurostats_api/config/tej_db *.yaml
+recursive-include neurostats_api/config/tej_db *.txt
+recursive-include neurostats_api/config/tej_db *.json
+recursive-include neurostats_api/config/twse *.yaml
+recursive-include neurostats_api/config/twse *.txt
+recursive-include neurostats_api/config/twse *.json
+recursive-include neurostats_api/config/company_list *.yaml
+recursive-include neurostats_api/config/company_list *.txt
+recursive-include neurostats_api/config/company_list *.json

neurostats_api-1.0.0rc2/PKG-INFO

@@ -0,0 +1,102 @@
+Metadata-Version: 2.1
+Name: neurostats_API
+Version: 1.0.0rc2
+Summary: The service of NeuroStats website
+Home-page: https://github.com/NeurowattStats/NeuroStats_API.git
+Author: JasonWang@Neurowatt
+Author-email: jason@neurowatt.ai
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pymongo
+Requires-Dist: pytz
+Requires-Dist: python-dotenv
+Requires-Dist: yfinance
+Requires-Dist: holidays
+
+# neurostats_API
+
+- [檔案架構](#檔案架構)
+- [下載](#下載)
+- [使用方法](#使用方法)
+
+## 檔案架構
+
+```
+./neurostats_API
+├── async_mode
+│   ├── db
+│   ├── db_extractors
+│   │   ├── daily
+│   │   ├── month_revenue
+│   │   └── seasonal
+│   ├── factory
+│   └── fetchers
+├── config
+│   ├── company_list
+│   ├── tej_db
+│   └── twse
+├── fetchers
+├── transformers
+│   ├── balance_sheet
+│   ├── cash_flow
+│   ├── daily_chip
+│   │   └── utils
+│   ├── daily_tech
+│   │   └── utils
+│   ├── finance_overview
+│   ├── month_revenue
+│   ├── profit_lose
+│   ├── tej
+│   └── value
+└── utils
+```
+- `neurostats_API`: 主要的package運行內容
+   - `async_mode`: \[新版新增\]以非同步方式進行DB資料的取得以及回傳資料
+      - `/db`: 連接到伺服器下的特定db
+      - `/db_extractor`: 決定取得db下的哪個collection的檔案，以及取得檔案內容
+      ` `/factory`: 使extractor的初始化變得通用
+      - `/fetchers`:  開發者將此套件用於其他專案的接口
+   - `fetchers`: (舊版)回傳service內容的fetcher檔案夾
+   - `transformers`: \[新版新增\] 處理從DB取出的資料，以dictionary的方式回傳需要的資料
+   - `config`: 【新舊通用】存放各種設定檔的dictionary，例如公司名單、各項指標對應的計算欄位、每個項目屬於哪份財報等設定
+   - `utils`: 【新舊通用】class之間通用的函式，例如讀取檔案、計算成長率函式、轉換格式與進位等
+
+## 下載
+```
+pip install neurostats-API
+```
+### 確認下載成功
+```Python 
+>>> import neurostats_API
+>>> print(neurostats_API.__version__)
+1.0.0rc2
+```
+
+### 下載舊版
+```
+pip install neurostats-API==0.0.25
+```
+
+## 使用方式 (以Balance_Sheet為例)
+### Async Mode
+```
+from neurostats_API.async_mode.fetchers import AsyncBalanceSheetFetcher
+from pymongo import AsyncMongoClient
+
+db_client = AsyncMongoClient("<伺服器的MONGO_URI>")
+ticker = "2330"
+
+fetcher = AsyncBalanceSheetFetcher(ticker, db_client)
+
+data = await fetcher.query_data()
+
+print(data)
+```
+詳細各個fetcher的使用法請到[async_mode的fetchers資料夾下觀看](./neurostats_API/async_mode/fetchers/README.md)
+
+### Sync Mode
+(#TODO)
+
+> 舊版使用方法請到[fetchers資料夾下觀看](./neurostats_API/fetchers/README.md)

neurostats_api-1.0.0rc2/README.md

@@ -0,0 +1,85 @@
+# neurostats_API
+
+- [檔案架構](#檔案架構)
+- [下載](#下載)
+- [使用方法](#使用方法)
+
+## 檔案架構
+
+```
+./neurostats_API
+├── async_mode
+│   ├── db
+│   ├── db_extractors
+│   │   ├── daily
+│   │   ├── month_revenue
+│   │   └── seasonal
+│   ├── factory
+│   └── fetchers
+├── config
+│   ├── company_list
+│   ├── tej_db
+│   └── twse
+├── fetchers
+├── transformers
+│   ├── balance_sheet
+│   ├── cash_flow
+│   ├── daily_chip
+│   │   └── utils
+│   ├── daily_tech
+│   │   └── utils
+│   ├── finance_overview
+│   ├── month_revenue
+│   ├── profit_lose
+│   ├── tej
+│   └── value
+└── utils
+```
+- `neurostats_API`: 主要的package運行內容
+   - `async_mode`: \[新版新增\]以非同步方式進行DB資料的取得以及回傳資料
+      - `/db`: 連接到伺服器下的特定db
+      - `/db_extractor`: 決定取得db下的哪個collection的檔案，以及取得檔案內容
+      ` `/factory`: 使extractor的初始化變得通用
+      - `/fetchers`:  開發者將此套件用於其他專案的接口
+   - `fetchers`: (舊版)回傳service內容的fetcher檔案夾
+   - `transformers`: \[新版新增\] 處理從DB取出的資料，以dictionary的方式回傳需要的資料
+   - `config`: 【新舊通用】存放各種設定檔的dictionary，例如公司名單、各項指標對應的計算欄位、每個項目屬於哪份財報等設定
+   - `utils`: 【新舊通用】class之間通用的函式，例如讀取檔案、計算成長率函式、轉換格式與進位等
+
+## 下載
+```
+pip install neurostats-API
+```
+### 確認下載成功
+```Python 
+>>> import neurostats_API
+>>> print(neurostats_API.__version__)
+1.0.0rc2
+```
+
+### 下載舊版
+```
+pip install neurostats-API==0.0.25
+```
+
+## 使用方式 (以Balance_Sheet為例)
+### Async Mode
+```
+from neurostats_API.async_mode.fetchers import AsyncBalanceSheetFetcher
+from pymongo import AsyncMongoClient
+
+db_client = AsyncMongoClient("<伺服器的MONGO_URI>")
+ticker = "2330"
+
+fetcher = AsyncBalanceSheetFetcher(ticker, db_client)
+
+data = await fetcher.query_data()
+
+print(data)
+```
+詳細各個fetcher的使用法請到[async_mode的fetchers資料夾下觀看](./neurostats_API/async_mode/fetchers/README.md)
+
+### Sync Mode
+(#TODO)
+
+> 舊版使用方法請到[fetchers資料夾下觀看](./neurostats_API/fetchers/README.md)

{neurostats_api-0.0.25rc1 → neurostats_api-1.0.0rc2}/neurostats_API/__init__.py

@@ -1,4 +1,4 @@
-__version__='0.0.25'
+__version__='1.0.0rc2'
 
 from .fetchers import (
     AgentFinanceOverviewFetcher,

neurostats_api-1.0.0rc2/neurostats_API/async_mode/__init__.py

@@ -0,0 +1,13 @@
+from .fetchers import (
+    AsyncAgentOverviewFetcher,
+    AsyncBalanceSheetFetcher,
+    AsyncCashFlowFetcher,
+    AsyncFinanceOverviewFetcher,
+    AsyncMonthlyRevenueFetcher,
+    AsyncProfitLoseFetcher,
+    AsyncTechFetcher,
+    AsyncTEJSeasonalFetcher,
+    AsyncTWSEInstitutionFetcher,
+    AsyncTWSEMarginFetcher,
+    AsyncTWSEStatsValueFetcher
+)

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db/__init__.py

@@ -0,0 +1,3 @@
+from .twse import TWSEDBClient
+from .tej import TEJDBClient
+from .us import USDBClient

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db/base.py

@@ -0,0 +1,24 @@
+from pymongo import AsyncMongoClient
+
+class BaseDBClient:
+    def __init__(self, client:  AsyncMongoClient, zone= None):
+        self.client = client
+        self.zone = zone or "tw"  # 預設台灣
+        self.db = self._select_db()
+
+    def _select_db(self):
+        """
+        根據 zone 選擇對應的資料庫
+        """
+        zone_db_map = {
+            "tw": "company_test",
+            "us": "company_us"
+        }
+        db_name = zone_db_map.get(self.zone, "company_test")
+        return self.client[db_name]
+
+    def get_collection(self, collection_name: str):
+        """
+        傳回指定 collection
+        """
+        return self.db[collection_name]

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db/tej.py

@@ -0,0 +1,10 @@
+from .base import BaseDBClient
+import os
+from pymongo import AsyncMongoClient
+from dotenv import load_dotenv
+
+class TEJDBClient(BaseDBClient):
+    def __init__(self, mongo_uri):
+        """初始化時接收 MongoDB 連接 URI"""
+        super().__init__(mongo_uri, zone = 'tw')
+        self.db = self.client["company_test"]

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db/twse.py

@@ -0,0 +1,8 @@
+from .base import BaseDBClient
+import os
+from pymongo import AsyncMongoClient
+from dotenv import load_dotenv
+
+class TWSEDBClient(BaseDBClient):
+    def __init__(self, client):
+        super().__init__(client, zone= "tw")

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db/us.py

@@ -0,0 +1,9 @@
+from .base import BaseDBClient
+import os
+from pymongo import AsyncMongoClient
+from dotenv import load_dotenv
+
+class USDBClient(BaseDBClient):
+    def __init__(self, client):
+        """初始化時接收 MongoDB 連接 URI"""
+        super().__init__(client, zone = 'us')

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/__init__.py

@@ -0,0 +1,20 @@
+from .daily import (
+    AsyncTEJDailyTechDBExtractor,
+    AsyncYFDailyTechDBExtractor,
+    AsyncDailyValueDBExtractor,
+    AsyncTEJDailyValueDBExtractor,
+    AsyncTWSEChipDBExtractor,
+
+)
+
+from .month_revenue import (
+    AsyncTWSEMonthlyRevenueExtractor
+)
+
+from .seasonal import (
+    AsyncBalanceSheetExtractor,
+    AsyncProfitLoseExtractor,
+    AsyncCashFlowExtractor,
+    AsyncTEJFinanceStatementDBExtractor,
+    AsyncTEJSelfSettlementDBExtractor
+)

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/base.py

@@ -0,0 +1,66 @@
+import abc
+from datetime import datetime
+from ..db import TWSEDBClient, USDBClient
+import json
+import pytz
+from neurostats_API.utils import StatsDateTime, StatsProcessor, YoY_Calculator, NoCompanyError
+
+class BaseDBExtractor(abc.ABC):
+    def __init__(self, ticker, client):
+        self.ticker = ticker
+        self.client = client
+        self.timezone = pytz.timezone("Asia/Taipei")
+        self.tw_company_list = StatsProcessor.load_json("company_list/tw.json")
+        self.us_company_list = StatsProcessor.load_json("company_list/us.json")
+
+        self.company_name, self.zone = self._set_company_name(self.ticker)
+
+        self.db_client = self._set_db_connection()
+
+    def _set_company_name(self, ticker):
+        company_lists = [
+            (self.tw_company_list, 'tw'),
+            (self.us_company_list, 'us')
+        ]
+
+        for company_list, zone in company_lists:
+            company_name = company_list.get(ticker, None)
+            if (company_name):
+                return company_name, zone
+        # 沒找到公司名稱
+        raise NoCompanyError(f"Ticker-{ticker} not found in any company lists")
+    
+    def _set_db_connection(self):
+        db_map = {
+            'tw': TWSEDBClient,
+            'us': USDBClient
+        }
+        db_client = db_map.get(self.zone, TWSEDBClient)
+        return db_client(self.client)
+
+
+    
+    async def query_data(self):
+        """
+        user使用的接口
+        """
+        return NotImplementedError()
+
+    @abc.abstractmethod
+    def _prepare_query(self):
+        pass
+    
+    def get_company_name(self):
+        """
+        回傳公司
+        """
+        return self.company_name
+    
+    def get_zone(self):
+        """
+        公司區域(TW, US)
+        """
+        return self.zone
+    
+    def _get_today(self):
+        return datetime.today()

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/daily/__init__.py

@@ -0,0 +1,7 @@
+from .tej_tech import AsyncTEJDailyTechDBExtractor
+from .twse_chip import AsyncTWSEChipDBExtractor
+from .value import (
+    AsyncDailyValueDBExtractor, 
+    AsyncTEJDailyValueDBExtractor
+)
+from .yf import AsyncYFDailyTechDBExtractor

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/daily/base.py

@@ -0,0 +1,89 @@
+from ..base import BaseDBExtractor
+from datetime import datetime
+from neurostats_API.utils import NotSupportedError
+from pymongo import ASCENDING, DESCENDING
+
+
+class BaseDailyTechDBExtractor(BaseDBExtractor):
+
+    def __init__(self, ticker, client):
+        super().__init__(ticker, client)
+
+        self.collection_name_map = None
+        self.collection_name = self._get_collection_name()
+        
+        if (self.collection_name is None):
+             raise NotSupportedError(
+                    f"{self.__class__.__name__} only supports {list(self.collection_name_map.keys())}, got {self.zone} with ticker = \"{self.ticker}\""
+                )
+
+        self.collection = self.db_client.get_collection(self.collection_name)
+
+    async def query_data(self, start_date=None, end_date=None, get_latest = False):
+        if (self.collection is None):
+            return []
+
+        if (start_date is None):
+            start_date = "1991-01-01"
+        if (end_date is None):
+            end_date = self._get_today()
+
+        query, projection, sort = self._prepare_query(
+            start_date=start_date, end_date=end_date, get_latest=get_latest
+        )
+
+        if (get_latest):
+            cursor = self.collection.find(query, projection).sort(sort).limit(1)
+        else:
+            cursor = self.collection.find(query, projection).sort(sort)
+
+        fetched_data = [data async for data in cursor]
+
+        return fetched_data
+
+    def _get_collection_name(self):
+        self.collection_name_map = {"tw": "twse_daily_share_price", "us": "us_technical"}
+        collection_name = self.collection_name_map.get(self.zone, None)
+
+        return collection_name 
+
+    def _prepare_query(self, start_date=None, end_date=None, get_latest = False):
+        query = {"ticker": self.ticker}
+
+        query = self._update_query_with_date(query, start_date, end_date)
+
+        projection = {
+            "_id": 0,
+        }
+
+        if (get_latest):
+            sort = [("date", DESCENDING)]
+        else:
+            sort = [("date", ASCENDING)]
+
+        return query, projection, sort
+
+    def _update_query_with_date(self, query, start_date, end_date):
+        start_date = self._transform_date(start_date)
+        end_date = self._transform_date(end_date)
+
+        date_range = {"$gte": start_date, "$lte": end_date}
+
+        query.update({"date": date_range})
+
+        return query
+
+    def _transform_date(self, date):
+        if (isinstance(date, str)):
+            try:
+                return datetime.strptime(date, "%Y-%m-%d")
+            except Exception as e:
+                raise ValueError(
+                    "date string format imcompatible, should be \"YYYY-MM-DD\""
+                )
+        elif (isinstance(date, datetime)):
+            return date
+        else:
+            raise ValueError(
+                "date type imcompatible, should be string(\"YYYY-MM-DD\") or datetime.datetime"
+            )

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/daily/tej_chip.py

@@ -0,0 +1,14 @@
+from .base import BaseDailyTechDBExtractor
+from datetime import datetime
+from pymongo import ASCENDING, DESCENDING
+
+
+class AsyncTEJDailyChipDBExtractor(BaseDailyTechDBExtractor):
+
+    def __init__(self, ticker, client):
+        super().__init__(ticker, client)
+
+    def _get_collection_name(self):
+        self.collection_name_map = {"tw": "TEJ_chip"}
+
+        return self.collection_name_map.get(self.zone, None)

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/daily/tej_tech.py

@@ -0,0 +1,12 @@
+from .base import BaseDailyTechDBExtractor
+
+
+class AsyncTEJDailyTechDBExtractor(BaseDailyTechDBExtractor):
+
+    def __init__(self, ticker, client):
+        super().__init__(ticker, client)
+
+    def _get_collection_name(self):
+        self.collection_name_map = {"tw": "TEJ_share_price"}
+
+        return self.collection_name_map.get(self.zone, None)

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/daily/twse_chip.py

@@ -0,0 +1,49 @@
+from .base import BaseDailyTechDBExtractor
+from datetime import datetime
+from pymongo import ASCENDING, DESCENDING
+from neurostats_API.utils import NotSupportedError
+
+class AsyncTWSEChipDBExtractor(BaseDailyTechDBExtractor):
+
+    def __init__(self, ticker, client, fetch_type = 'I'):
+
+        """
+        擷取台股每日籌碼資料。
+
+        fetch_type:
+            - 'I' (INSTITUTION): 法人買賣
+            - 'M' (MARGIN): 融資融券餘額變化
+            - 'S' (SECURITY_LENDING): 借券債券
+
+        """
+        super().__init__(ticker, client)
+
+        column_name_map = {
+            'tw': {
+                "I" : "institution_trading",
+                "M" : "margin_trading",
+                "S" : "security_lending"
+            },
+        }
+        self.target_column = column_name_map[self.zone][fetch_type]
+
+    def _get_collection_name(self):
+        self.collection_name_map = {"tw": "twse_chip"}
+        collection_name = self.collection_name_map.get(self.zone, None)
+        return collection_name
+    
+    def _prepare_query(self, start_date=None, end_date=None, get_latest = False):
+        query, projection, sort = super()._prepare_query(start_date, end_date, get_latest)
+        query.update(
+            {
+                self.target_column: {"$exists" : True}
+            }
+        )
+        projection.update(
+            {
+                'date': 1,
+                self.target_column: 1
+            }
+        )
+
+        return query, projection, sort

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/daily/value.py

@@ -0,0 +1,93 @@
+from .base import BaseDailyTechDBExtractor
+from pymongo import ASCENDING, DESCENDING
+from neurostats_API.utils import NotSupportedError
+
+class AsyncDailyValueDBExtractor(BaseDailyTechDBExtractor):
+
+    def __init__(self, ticker, client, fetch_type='D'):
+        """
+        fetch_type: 
+        'D': daily
+        'Y': yearly
+        """
+        self.fetch_type = fetch_type
+        super().__init__(ticker, client)
+        
+    def _get_collection_name(self):
+        self.collection_name_map = {
+            "tw": {
+                "D":"twse_daily_share_price",
+                "Y": "twse_yearly_value"
+            }
+        }
+        collection_map = self.collection_name_map.get(self.zone)
+        if (collection_map is None):
+            raise NotSupportedError(f"AsyncDailyValueDBExtractor only supports {self.collection_name_map.keys}, but get {self.zone}")
+        collection_name = collection_map.get(self.fetch_type)
+        if (collection_name is None):
+            raise ValueError(f"AsyncDailyValueDBExtractor: Invalid fetch type : {self.fetch_type}")
+        return collection_name
+    
+    def _prepare_query(self, start_date = None, end_date = None, get_latest = False):
+        query_fn_map = {
+            'D': super()._prepare_query,
+            'Y': self._prepare_annual_query
+        }
+
+        query_fn = query_fn_map.get(self.fetch_type)
+        query, projection, sorting = query_fn(start_date=start_date, end_date=end_date, get_latest=get_latest)
+
+        projection.update(
+            {   
+                "date": 1,
+                "year": 1,
+                "close": 1,
+                "P_E": 1,
+                "P_B": 1,
+                "P_FCF": 1,
+                "P_S": 1,
+                "EV_S": 1,
+                "EV_OPI": 1,
+                "EV_EBIT": 1,
+                "EV_EBITDA": 1
+            }
+        )
+
+        return query, projection, sorting
+
+
+    def _prepare_annual_query(self, start_date, end_date, get_latest = False):
+        start_date = self._transform_date(start_date)
+        end_date = self._transform_date(end_date)
+
+        start_year = start_date.year - 1911
+        end_year = end_date.year - 1911
+        query = {
+            "ticker": self.ticker,
+            "year": {
+                "$gte": start_year,
+                "$lte": end_year
+            }
+        }
+
+        projection = {
+            "_id": 0,
+        }
+
+        if (get_latest):
+            sort = [("year", DESCENDING)]
+        else:
+            sort = [("year", ASCENDING)]
+
+        return query, projection, sort
+
+
+class AsyncTEJDailyValueDBExtractor(BaseDailyTechDBExtractor):
+
+    def __init__(self, ticker, mongo_uri):
+        super().__init__(ticker, mongo_uri)
+
+    def _get_collection_name(self):
+        self.collection_name_map = {"tw": "TEJ_share_price"}
+
+        return self.collection_name_map.get(self.zone, None)

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/daily/yf.py

@@ -0,0 +1,12 @@
+from .base import BaseDailyTechDBExtractor
+
+
+class AsyncYFDailyTechDBExtractor(BaseDailyTechDBExtractor):
+
+    def __init__(self, ticker, client):
+        super().__init__(ticker, client)
+
+    def _get_collection_name(self):
+        self.collection_name_map = {"tw": "twse_daily_share_price", "us": "us_technical"}
+
+        return self.collection_name_map.get(self.zone, None)

neurostats_api-1.0.0rc2/neurostats_API/async_mode/db_extractors/month_revenue/__init__.py

@@ -0,0 +1 @@
+from .twse import AsyncTWSEMonthlyRevenueExtractor