FinMind 1.2.8__tar.gz → 2.0.0__tar.gz

finmind-2.0.0/FinMind/__init__.py

@@ -0,0 +1,2 @@
+from FinMind import strategies, crawler, data, schema
+from FinMind._version import __version__

{FinMind-1.2.8 → finmind-2.0.0}/FinMind/_version.py

@@ -1,6 +1,5 @@
 from importlib_metadata import version, PackageNotFoundError
 
-
 try:
     __version__ = version(__name__.replace("._version", ""))
 except PackageNotFoundError:

finmind-2.0.0/FinMind/crawler/__init__.py

@@ -0,0 +1,2 @@
+from FinMind.crawler.commodities import CommoditiesCrawler
+from FinMind.crawler.government_bonds import GovernmentBondsCrawler

FinMind-1.2.8/FinMind/Crawler/BasedClass.py → finmind-2.0.0/FinMind/crawler/base.py

@@ -1,7 +1,13 @@
 import datetime
+import time
+
 import numpy as np
 import requests
-import time
+
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/68.0.3440.84 Safari/537.36"
+)
 
 
 def requests_get(url, header):
@@ -29,15 +35,17 @@ def get_time():
     return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
 
 
-class Crawler:
-    def date2days(self, date):
+class BaseCrawler:
+    @staticmethod
+    def date2days(date):
         # date = '2018-08-03'
         date = datetime.datetime.strptime(date, "%Y-%m-%d").date()
         value = (date - datetime.date(1970, 1, 1)).days
         value = value * 60 * 60 * 24 * 1000
         return value
 
-    def days2date(self, day):
+    @staticmethod
+    def days2date(day):
         # day = 631497600000
         # 60s = 1min
         # 60min = 1hr
@@ -103,7 +111,8 @@ class Crawler:
         value = "{} {}:{}:{}".format(date, hour, minute, second)
         return value
 
-    def date2millisecond(self, date):
+    @staticmethod
+    def date2millisecond(date):
         # date = '2019-06-02 15:30:00'
         date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
         date = date + datetime.timedelta(minutes=1)
@@ -114,7 +123,8 @@ class Crawler:
 
         return second
 
-    def create_date(self, start, today=False):  # start = '2018-07-31'
+    @staticmethod
+    def create_date(start, today=False):  # start = '2018-07-31'
         start = datetime.datetime.strptime(
             start, "%Y-%m-%d"
         ).date() + datetime.timedelta(days=1)
@@ -128,7 +138,8 @@ class Crawler:
         ]
         return date
 
-    def remove_outlier(self, data, var_name):
+    @staticmethod
+    def remove_outlier(data, var_name):
 
         value = list(data[var_name])
         mean = np.mean(value, axis=0)

FinMind-1.2.8/FinMind/Crawler/RawMaterialFuturesPrices.py → finmind-2.0.0/FinMind/crawler/commodities.py

@@ -1,27 +1,24 @@
-TABLE = "RawMaterialFuturesPrices"
-
+import datetime
 import os
-import requests
+import re
 import sys
+
 import pandas as pd
-import datetime
-import re
+import requests
 from lxml import etree
 
+from FinMind.crawler.base import BaseCrawler, USER_AGENT
+
 PATH = "/".join(os.path.abspath(__file__).split("/")[:-2])
 sys.path.append(PATH)
-import Crawler.BasedClass as cbaseclass
-
-"""
-self = Crawler()
-"""
 
 
-class Crawler(cbaseclass.Crawler):
+class CommoditiesCrawler(BaseCrawler):
     def __init__(self):
-        super(Crawler, self).__init__()
+        super(CommoditiesCrawler, self).__init__()
 
-    def create_loop_list(self):
+    @staticmethod
+    def create_loop_list():
         # self.based_url = 'https://www.investing.com/commodities/'
         kind_list = ["meats", "grains", "energies", "softs", "metals"]
         loop_list = []
@@ -36,7 +33,7 @@ class Crawler(cbaseclass.Crawler):
                 "Connection": "keep-alive",
                 "Host": "www.investing.com",
                 "Upgrade-Insecure-Requests": "1",
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36",
+                "User-Agent": USER_AGENT,
             }
             res = requests.get(index_url, verify=True, headers=headers)
 
@@ -53,11 +50,8 @@ class Crawler(cbaseclass.Crawler):
             ]
         return loop_list
 
-    def get_st_date(self, futures_id, data_name):
-        return "01/01/1970"
-        # return '01/01/2018'
-
-    def get_end_date(self):
+    @staticmethod
+    def get_end_date():
         end_date = datetime.datetime.now().date()
         end_date = end_date + datetime.timedelta(-1)
         y = str(end_date.year)
@@ -71,30 +65,36 @@ class Crawler(cbaseclass.Crawler):
         return m + "/" + d + "/" + y
 
     def crawler(self, loop):  # loop = ['49769', 'Brent Oil Futures']
-        def get_value(tem):
+        def get_value(template):
 
-            date = int(tem[0].attrib["data-real-value"])
+            date = int(template[0].attrib["data-real-value"])
             date = int(date / 60 / 60 / 24)
             date = str(
                 datetime.date(1970, 1, 1) + datetime.timedelta(days=date)
             )
             v = [
-                float(tem[i].attrib["data-real-value"].replace(",", ""))
-                for i in range(1, 6)
+                float(
+                    template[template_index]
+                    .attrib["data-real-value"]
+                    .replace(",", "")
+                )
+                for template_index in range(1, 6)
             ]
-            price, Open, High, Low, Vol = v
-
-            Change = float(tem[6].text.replace("%", "").replace(",", "")) / 100
+            _price, _open, _high, _low, _vol = v
 
-            data = pd.DataFrame([date, price, Open, High, Low, Vol, Change]).T
+            change = (
+                float(template[6].text.replace("%", "").replace(",", "")) / 100
+            )
 
-            return data
+            return pd.DataFrame(
+                [date, _price, _open, _high, _low, _vol, change]
+            ).T
 
         # -------------------------------------------------------------------
         futures_id, data_name = loop
-        header = data_name + " Historical Data"
+        header = data_name + " Historical data"
         st_date, end_date = (
-            self.get_st_date(futures_id, data_name),
+            "01/01/1970",
             self.get_end_date(),
         )
 
@@ -120,7 +120,7 @@ class Crawler(cbaseclass.Crawler):
             "Host": "www.investing.com",
             "Origin": "https://www.investing.com",
             "Referer": "https://www.investing.com/commodities/brent-oil-historical-data",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36",
+            "User-Agent": USER_AGENT,
             "X-Requested-With": "XMLHttpRequest",
         }
         print("requests post")
@@ -129,22 +129,22 @@ class Crawler(cbaseclass.Crawler):
         )
         print("data clean")
         page = etree.HTML(res.text)
-        colname = page.xpath("//tr//th")
+        col_name = page.xpath("//tr//th")
 
-        colname = [
-            c.text.replace(" %", "Percent").replace(".", "") for c in colname
+        col_name = [
+            c.text.replace(" %", "Percent").replace(".", "") for c in col_name
         ]
-        colname = ["date" if c == "Date" else c for c in colname]
+        col_name = ["date" if c == "Date" else c for c in col_name]
 
         data = pd.DataFrame()
         td_path = page.xpath("//tr//td")
         for i in range(0, len(td_path) - 7, 7):
             tem = td_path[i : i + 7]
             value = get_value(tem)
-            data = data.append(value)
+            data = pd.concat([data, value])
 
         if len(data) > 0:
-            data.columns = colname
+            data.columns = col_name
             data["name"] = data_name
             # data['data_id'] = futures_id
             data = data.sort_values("date")

finmind-2.0.0/FinMind/crawler/demo.py

@@ -0,0 +1,32 @@
+import pandas as pd
+
+from FinMind.crawler import GovernmentBondsCrawler, CommoditiesCrawler
+
+# -------------------------------------------------------------------
+commodities_crawler = CommoditiesCrawler()
+# get futures list
+loop_list = commodities_crawler.create_loop_list()
+
+# get one futures data
+commodities_df = commodities_crawler.crawler(loop_list[0])
+
+# or get all futures data
+commodities_df = pd.DataFrame()
+for loop in loop_list:
+    print(loop)
+    value = commodities_crawler.crawler(loop)
+    commodities_df = pd.concat([commodities_df, value])
+# -------------------------------------------------------------------
+gb_crawler = GovernmentBondsCrawler()
+# get futures list
+loop_list = gb_crawler.create_loop_list()
+
+# get one futures data
+gd_df = gb_crawler.crawler(loop_list[0])
+
+# or get all futures data
+gd_df = pd.DataFrame()
+for loop in loop_list:
+    print(loop)
+    value = gb_crawler.crawler(loop)
+    gd_df = pd.concat([gd_df, value])

FinMind-1.2.8/FinMind/Crawler/GovernmentBonds.py → finmind-2.0.0/FinMind/crawler/government_bonds.py

@@ -1,30 +1,27 @@
 """
-G8
-俄羅斯、美國、加拿大、英國、法國、德國、義大利及日本
+政府債券
+G8-俄羅斯、美國、加拿大、英國、法國、德國、義大利及日本
 """
-TABLE = "GovernmentBonds"
+
+import datetime
 import os
-import requests
+import re
 import sys
+
 import pandas as pd
-import re
-import datetime
+import requests
 from lxml import etree
 
+from FinMind.crawler.base import BaseCrawler, USER_AGENT
+
 PATH = "/".join(os.path.abspath(__file__).split("/")[:-2])
 sys.path.append(PATH)
-import Crawler.BasedClass as cbaseclass
 
 
-"""
-self = Crawler()
-
-"""
-
-
-class Crawler(cbaseclass.Crawler):
-    def create_loop_list(self):
-        def get_data_id_name(curl):
+class GovernmentBondsCrawler(BaseCrawler):
+    @staticmethod
+    def create_loop_list():
+        def get_data_id_name(url):
 
             headers = {
                 "Accept": "*/*",
@@ -32,24 +29,24 @@ class Crawler(cbaseclass.Crawler):
                 "Accept-Language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
                 "Connection": "keep-alive",
                 "Host": "www.investing.com",
-                "Referer": curl,
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36",
+                "Referer": url,
+                "User-Agent": USER_AGENT,
                 "X-Requested-With": "XMLHttpRequest",
             }
 
-            res = requests.get(curl, verify=True, headers=headers)
+            res = requests.get(url, verify=True, headers=headers)
             tem_data_id = re.findall('data-id="[0-9]+"', res.text)
             tem_data_id = [di.replace("data-id=", "") for di in tem_data_id]
             page = etree.HTML(res.text)
-            data_id = []
-            data_name = []
+            _data_id = []
+            _data_name = []
             for di in tem_data_id:
                 tem = page.xpath("//span[@data-id={}]".format(di))
                 if len(tem) > 0:
-                    data_id.append(tem[0].attrib["data-id"])
-                    data_name.append(tem[0].attrib["data-name"])
+                    _data_id.append(tem[0].attrib["data-id"])
+                    _data_name.append(tem[0].attrib["data-name"])
 
-            return data_id, data_name
+            return _data_id, _data_name
 
         def get_country_url():
             index_url = "https://www.investing.com/rates-bonds/"
@@ -61,7 +58,7 @@ class Crawler(cbaseclass.Crawler):
                 "Connection": "keep-alive",
                 "Host": "www.investing.com",
                 "Upgrade-Insecure-Requests": "1",
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36",
+                "User-Agent": USER_AGENT,
             }
             res = requests.get(index_url, verify=True, headers=headers)
 
@@ -91,15 +88,15 @@ class Crawler(cbaseclass.Crawler):
                 "usa",
                 "italy",
             ]
-            country_url = []
-            for i in range(len(url)):
-                tem = url[i].replace(
+            countries_url = []
+            for url_index in range(len(url)):
+                tem = url[url_index].replace(
                     "https://www.investing.com/rates-bonds/", ""
                 )
                 tem = tem.replace("-government-bonds", "")
                 if tem in select:
-                    country_url.append(url[i])
-            return country_url
+                    countries_url.append(url[url_index])
+            return countries_url
 
         # main
         country_url = get_country_url()
@@ -112,10 +109,8 @@ class Crawler(cbaseclass.Crawler):
 
         return loop_list
 
-    def get_st_date(self, data_name, cid):
-        return "01/01/1970"
-
-    def get_end_date(self):
+    @staticmethod
+    def get_end_date():
 
         end_date = datetime.datetime.now().date()
         end_date = end_date + datetime.timedelta(-1)
@@ -130,27 +125,31 @@ class Crawler(cbaseclass.Crawler):
         return "{}/{}/{}".format(m, d, y)
 
     def crawler(self, loop):  # loop = ['23681', 'Germany 3 Month']
-        def get_value(tem):
+        def get_value(template):
 
-            date = int(tem[0].attrib["data-real-value"])
+            date = int(template[0].attrib["data-real-value"])
             date = int(date / 60 / 60 / 24)
             date = str(
                 datetime.date(1970, 1, 1) + datetime.timedelta(days=date)
             )
-            v = [float(tem[i].text) for i in range(1, 5) if tem[i].text != None]
+            v = [
+                float(template[template_index].text)
+                for template_index in range(1, 5)
+                if template[template_index].text is not None
+            ]
             if len(v) == 0:
                 return pd.DataFrame()
-            price, Open, High, Low = v
-
-            Change = float(tem[5].text.replace("%", "").replace(",", "")) / 100
+            _price, _open, _high, _low = v
+            change = (
+                float(template[5].text.replace("%", "").replace(",", "")) / 100
+            )
 
-            data = pd.DataFrame([date, price, Open, High, Low, Change]).T
-            return data
+            return pd.DataFrame([date, _price, _open, _high, _low, change]).T
 
         cid, data_name = loop
-        header = data_name + " Bond Yield Historical Data"
+        header = data_name + " Bond Yield Historical data"
         st_date, end_date = (
-            self.get_st_date(data_name, cid),
+            "01/01/1970",
             self.get_end_date(),
         )
         bonds_url = "https://www.investing.com/instruments/HistoricalDataAjax"
@@ -176,7 +175,7 @@ class Crawler(cbaseclass.Crawler):
             "Host": "www.investing.com",
             "Origin": "https://www.investing.com",
             "Referer": "https://www.investing.com/rates-bonds/france-1-month-bond-yield-historical-data",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36",
+            "User-Agent": USER_AGENT,
             "X-Requested-With": "XMLHttpRequest",
         }
         print("requests post")
@@ -187,19 +186,19 @@ class Crawler(cbaseclass.Crawler):
         page = etree.HTML(res.text)
         tr_path = page.xpath("//tr")
 
-        colname = [col.text for col in tr_path[0].xpath("//th")]
-        colname = [c.replace(" %", "Percent") for c in colname]
-        colname = ["date" if c == "Date" else c for c in colname]
+        col_name = [col.text for col in tr_path[0].xpath("//th")]
+        col_name = [c.replace(" %", "Percent") for c in col_name]
+        col_name = ["date" if c == "Date" else c for c in col_name]
         data = pd.DataFrame()
         td_path = page.xpath("//tr//td")
         for i in range(0, len(td_path) - 6, 6):
             tem = td_path[i : i + 6]
             value = get_value(tem)
             if len(value) > 0:
-                data = data.append(value)
+                data = pd.concat([data, value])
 
         if len(data) > 0:
-            data.columns = colname
+            data.columns = col_name
             data["name"] = "{}".format(data_name)
             # data['data_id'] = cid
             data = data.sort_values("date")

finmind-2.0.0/FinMind/data/__init__.py

@@ -0,0 +1,5 @@
+from FinMind.data.finmind_api import FinMindApi
+from FinMind.data.data_loader import DataLoader
+from FinMind.data.data_subscriber import DataSubscriber
+from FinMind.data.data_subscriber import Stock
+from FinMind.data.data_subscriber import FutureAndOption