finmind 1.8.0__py3-none-any.whl

FinMind/__init__.py

@@ -0,0 +1,2 @@
+from FinMind import strategies, crawler, data, schema
+from FinMind._version import __version__

FinMind/_version.py

@@ -0,0 +1,6 @@
+from importlib_metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version(__name__.replace("._version", ""))
+except PackageNotFoundError:
+    __version__ = "0.0.0.nof"

FinMind/crawler/__init__.py

@@ -0,0 +1,2 @@
+from FinMind.crawler.commodities import CommoditiesCrawler
+from FinMind.crawler.government_bonds import GovernmentBondsCrawler

FinMind/crawler/base.py

@@ -0,0 +1,165 @@
+import datetime
+import time
+
+import numpy as np
+import requests
+
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/68.0.3440.84 Safari/537.36"
+)
+
+
+def requests_get(url, header):
+    try:
+        res = requests.get(url, verify=False, timeout=10, headers=header)
+        return res
+    except Exception as e:
+        if "Max retries exceeded" in str(e) or "Read timed out" in str(e):
+            time.sleep(60)
+
+
+def requests_post(url, header, form_data):
+    try:
+        res = requests.post(
+            url, verify=False, timeout=10, headers=header, data=form_data
+        )
+        return res
+    except Exception as e:
+        if "Max retries exceeded" in str(e) or "Read timed out" in str(e):
+            time.sleep(60)
+            raise
+
+
+def get_time():
+    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
+
+
+class BaseCrawler:
+    @staticmethod
+    def date2days(date):
+        # date = '2018-08-03'
+        date = datetime.datetime.strptime(date, "%Y-%m-%d").date()
+        value = (date - datetime.date(1970, 1, 1)).days
+        value = value * 60 * 60 * 24 * 1000
+        return value
+
+    @staticmethod
+    def days2date(day):
+        # day = 631497600000
+        # 60s = 1min
+        # 60min = 1hr
+        day = int(day)
+        day = int(day / 1000 / 60 / 60 / 24)
+        value = datetime.date(1970, 1, 1) + datetime.timedelta(days=day)
+        return value
+
+    def millisecond2date(self, ms):
+        # ms = 1559489350000
+        ms = int(ms)
+        date = str(self.days2date(ms))
+        days = int(ms / 1000 / 60 / 60 / 24)
+
+        second = ms / 1000 - days * 60 * 60 * 24
+        hour = int(second / 60 / 60)
+        minute = int((second - hour * 60 * 60) / 60) - 1
+
+        if hour < 0:
+            hour = "00"
+        elif hour < 10:
+            hour = "0" + str(hour)
+        else:
+            hour = str(hour)
+
+        if minute < 0:
+            minute = "00"
+        elif minute < 10:
+            minute = "0" + str(minute)
+        else:
+            minute = str(minute)
+        # second = (second - hour*60*60 - minute*60)#hour
+        value = date + " " + hour + ":" + minute + ":00"
+        return value
+
+    def millisecond2date2(self, ms):
+        # ms = 1566898740000
+        ms = int(ms)
+        date = str(self.days2date(ms))
+        days = int(ms / 1000 / 60 / 60 / 24)
+
+        second = ms / 1000 - days * 60 * 60 * 24
+        hour = int(second / 60 / 60)
+        minute = int((second - hour * 60 * 60) / 60)
+        second = int(second - hour * 60 * 60 - minute * 60)
+
+        if hour < 0:
+            hour = "00"
+        elif hour < 10:
+            hour = "0" + str(hour)
+        else:
+            hour = str(hour)
+
+        if minute < 0:
+            minute = "00"
+        elif minute < 10:
+            minute = "0" + str(minute)
+        else:
+            minute = str(minute)
+        # second = (second - hour*60*60 - minute*60)#hour
+        if second < 10:
+            second = "0{}".format(second)
+        value = "{} {}:{}:{}".format(date, hour, minute, second)
+        return value
+
+    @staticmethod
+    def date2millisecond(date):
+        # date = '2019-06-02 15:30:00'
+        date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
+        date = date + datetime.timedelta(minutes=1)
+        second = date - datetime.datetime(1970, 1, 1, 0, 0, 0)
+
+        second = second.days * 24 * 60 * 60 + second.seconds
+        # ms = ms*1000
+
+        return second
+
+    @staticmethod
+    def create_date(start, today=False):  # start = '2018-07-31'
+        start = datetime.datetime.strptime(
+            start, "%Y-%m-%d"
+        ).date() + datetime.timedelta(days=1)
+        end = datetime.date.today()
+
+        day_len = (end - start).days
+        if today:
+            day_len = (end - start).days + 1
+        date = [
+            str(start + datetime.timedelta(days=dat)) for dat in range(day_len)
+        ]
+        return date
+
+    @staticmethod
+    def remove_outlier(data, var_name):
+
+        value = list(data[var_name])
+        mean = np.mean(value, axis=0)
+        sd = np.std(value, axis=0)
+        if sd < 1:
+            return data
+
+        _bool = []
+        for x in value:
+            if (5 * mean) > x > (-5 * mean):
+                _bool.append(True)
+            else:
+                _bool.append(False)
+
+        data = data[_bool]
+        return data
+
+
+def change_chinese_date_us(d):
+    y, m, d = [int(x) for x in d.split("/")]
+    y = y + 1911
+    date = datetime.date(y, m, d)
+    return date

FinMind/crawler/commodities.py

@@ -0,0 +1,153 @@
+import datetime
+import os
+import re
+import sys
+
+import pandas as pd
+import requests
+from lxml import etree
+
+from FinMind.crawler.base import BaseCrawler, USER_AGENT
+
+PATH = "/".join(os.path.abspath(__file__).split("/")[:-2])
+sys.path.append(PATH)
+
+
+class CommoditiesCrawler(BaseCrawler):
+    def __init__(self):
+        super(CommoditiesCrawler, self).__init__()
+
+    @staticmethod
+    def create_loop_list():
+        # self.based_url = 'https://www.investing.com/commodities/'
+        kind_list = ["meats", "grains", "energies", "softs", "metals"]
+        loop_list = []
+        for kind in kind_list:
+            print(kind)
+            index_url = "https://www.investing.com/commodities/" + kind
+            headers = {
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Accept-Language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+                "Cache-Control": "max-age=0",
+                "Connection": "keep-alive",
+                "Host": "www.investing.com",
+                "Upgrade-Insecure-Requests": "1",
+                "User-Agent": USER_AGENT,
+            }
+            res = requests.get(index_url, verify=True, headers=headers)
+
+            tem = re.findall(
+                ' data-name="[A-Za-z ]+" data-id="[0-9]+" ', res.text
+            )
+            futures_id_list = [re.findall("[0-9]+", te)[0] for te in tem]
+            data_name_list = [
+                re.findall('"[A-Za-z ]+"', te)[0].replace('"', "") for te in tem
+            ]
+            [
+                loop_list.append([futures_id_list[i], data_name_list[i]])
+                for i in range(len(futures_id_list))
+            ]
+        return loop_list
+
+    @staticmethod
+    def get_end_date():
+        end_date = datetime.datetime.now().date()
+        end_date = end_date + datetime.timedelta(-1)
+        y = str(end_date.year)
+        m = (
+            str(end_date.month)
+            if end_date.month > 9
+            else "0" + str(end_date.month)
+        )
+        d = str(end_date.day) if end_date.day > 9 else "0" + str(end_date.day)
+
+        return m + "/" + d + "/" + y
+
+    def crawler(self, loop):  # loop = ['49769', 'Brent Oil Futures']
+        def get_value(template):
+
+            date = int(template[0].attrib["data-real-value"])
+            date = int(date / 60 / 60 / 24)
+            date = str(
+                datetime.date(1970, 1, 1) + datetime.timedelta(days=date)
+            )
+            v = [
+                float(
+                    template[template_index]
+                    .attrib["data-real-value"]
+                    .replace(",", "")
+                )
+                for template_index in range(1, 6)
+            ]
+            _price, _open, _high, _low, _vol = v
+
+            change = (
+                float(template[6].text.replace("%", "").replace(",", "")) / 100
+            )
+
+            return pd.DataFrame(
+                [date, _price, _open, _high, _low, _vol, change]
+            ).T
+
+        # -------------------------------------------------------------------
+        futures_id, data_name = loop
+        header = data_name + " Historical data"
+        st_date, end_date = (
+            "01/01/1970",
+            self.get_end_date(),
+        )
+
+        bonds_url = "https://www.investing.com/instruments/HistoricalDataAjax"
+        form_data = {
+            "curr_id": futures_id,
+            "header": header,
+            "st_date": st_date,
+            "end_date": end_date,
+            "interval_sec": "Daily",
+            "sort_col": "date",
+            "sort_ord": "DESC",
+            "action": "historical_data",
+        }
+
+        headers = {
+            "Accept": "text/plain, */*; q=0.01",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Accept-Language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+            "Connection": "keep-alive",
+            "Content-Length": "183",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Host": "www.investing.com",
+            "Origin": "https://www.investing.com",
+            "Referer": "https://www.investing.com/commodities/brent-oil-historical-data",
+            "User-Agent": USER_AGENT,
+            "X-Requested-With": "XMLHttpRequest",
+        }
+        print("requests post")
+        res = requests.post(
+            bonds_url, verify=True, headers=headers, data=form_data
+        )
+        print("data clean")
+        page = etree.HTML(res.text)
+        col_name = page.xpath("//tr//th")
+
+        col_name = [
+            c.text.replace(" %", "Percent").replace(".", "") for c in col_name
+        ]
+        col_name = ["date" if c == "Date" else c for c in col_name]
+
+        data = pd.DataFrame()
+        td_path = page.xpath("//tr//td")
+        for i in range(0, len(td_path) - 7, 7):
+            tem = td_path[i : i + 7]
+            value = get_value(tem)
+            data = data.append(value)
+
+        if len(data) > 0:
+            data.columns = col_name
+            data["name"] = data_name
+            # data['data_id'] = futures_id
+            data = data.sort_values("date")
+            data.index = range(len(data))
+
+        return data

FinMind/crawler/demo.py

@@ -0,0 +1,32 @@
+import pandas as pd
+
+from FinMind.crawler import GovernmentBondsCrawler, CommoditiesCrawler
+
+# -------------------------------------------------------------------
+commodities_crawler = CommoditiesCrawler()
+# get futures list
+loop_list = commodities_crawler.create_loop_list()
+
+# get one futures data
+commodities_df = commodities_crawler.crawler(loop_list[0])
+
+# or get all futures data
+commodities_df = pd.DataFrame()
+for loop in loop_list:
+    print(loop)
+    value = commodities_crawler.crawler(loop)
+    commodities_df = commodities_df.append(value)
+# -------------------------------------------------------------------
+gb_crawler = GovernmentBondsCrawler()
+# get futures list
+loop_list = gb_crawler.create_loop_list()
+
+# get one futures data
+gd_df = gb_crawler.crawler(loop_list[0])
+
+# or get all futures data
+gd_df = pd.DataFrame()
+for loop in loop_list:
+    print(loop)
+    value = gb_crawler.crawler(loop)
+    gd_df = gd_df.append(value)

FinMind/crawler/government_bonds.py

@@ -0,0 +1,207 @@
+"""
+政府債券
+G8-俄羅斯、美國、加拿大、英國、法國、德國、義大利及日本
+"""
+
+import datetime
+import os
+import re
+import sys
+
+import pandas as pd
+import requests
+from lxml import etree
+
+from FinMind.crawler.base import BaseCrawler, USER_AGENT
+
+PATH = "/".join(os.path.abspath(__file__).split("/")[:-2])
+sys.path.append(PATH)
+
+
+class GovernmentBondsCrawler(BaseCrawler):
+    @staticmethod
+    def create_loop_list():
+        def get_data_id_name(url):
+
+            headers = {
+                "Accept": "*/*",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Accept-Language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+                "Connection": "keep-alive",
+                "Host": "www.investing.com",
+                "Referer": url,
+                "User-Agent": USER_AGENT,
+                "X-Requested-With": "XMLHttpRequest",
+            }
+
+            res = requests.get(url, verify=True, headers=headers)
+            tem_data_id = re.findall('data-id="[0-9]+"', res.text)
+            tem_data_id = [di.replace("data-id=", "") for di in tem_data_id]
+            page = etree.HTML(res.text)
+            _data_id = []
+            _data_name = []
+            for di in tem_data_id:
+                tem = page.xpath("//span[@data-id={}]".format(di))
+                if len(tem) > 0:
+                    _data_id.append(tem[0].attrib["data-id"])
+                    _data_name.append(tem[0].attrib["data-name"])
+
+            return _data_id, _data_name
+
+        def get_country_url():
+            index_url = "https://www.investing.com/rates-bonds/"
+            headers = {
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Accept-Language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+                "Cache-Control": "max-age=0",
+                "Connection": "keep-alive",
+                "Host": "www.investing.com",
+                "Upgrade-Insecure-Requests": "1",
+                "User-Agent": USER_AGENT,
+            }
+            res = requests.get(index_url, verify=True, headers=headers)
+
+            data_country_id = re.findall('data-country-id="[0-9]+"', res.text)
+            data_country_id = [
+                dci.replace("data-country-id=", "") for dci in data_country_id
+            ]
+            page = etree.HTML(res.text)
+            tem = []
+            for dci in data_country_id:
+                tem.append(
+                    page.xpath("//option[@data-country-id={}]".format(dci))[0]
+                )
+
+            url = [
+                "https://www.investing.com" + te.attrib["value"] for te in tem
+            ]
+            # G8 and china
+            select = [
+                "canada",
+                "china",
+                "france",
+                "germany",
+                "japan",
+                "russia",
+                "uk",
+                "usa",
+                "italy",
+            ]
+            countries_url = []
+            for url_index in range(len(url)):
+                tem = url[url_index].replace(
+                    "https://www.investing.com/rates-bonds/", ""
+                )
+                tem = tem.replace("-government-bonds", "")
+                if tem in select:
+                    countries_url.append(url[url_index])
+            return countries_url
+
+        # main
+        country_url = get_country_url()
+        loop_list = []
+        for curl in country_url:  # curl = country_url[0]
+            print(curl)
+            data_id, data_name = get_data_id_name(curl)
+            for i in range(len(data_id)):
+                loop_list.append([data_id[i], data_name[i]])
+
+        return loop_list
+
+    @staticmethod
+    def get_end_date():
+
+        end_date = datetime.datetime.now().date()
+        end_date = end_date + datetime.timedelta(-1)
+        y = str(end_date.year)
+        m = (
+            str(end_date.month)
+            if end_date.month > 9
+            else "0" + str(end_date.month)
+        )
+        d = str(end_date.day) if end_date.day > 9 else "0" + str(end_date.day)
+
+        return "{}/{}/{}".format(m, d, y)
+
+    def crawler(self, loop):  # loop = ['23681', 'Germany 3 Month']
+        def get_value(template):
+
+            date = int(template[0].attrib["data-real-value"])
+            date = int(date / 60 / 60 / 24)
+            date = str(
+                datetime.date(1970, 1, 1) + datetime.timedelta(days=date)
+            )
+            v = [
+                float(template[template_index].text)
+                for template_index in range(1, 5)
+                if template[template_index].text is not None
+            ]
+            if len(v) == 0:
+                return pd.DataFrame()
+            _price, _open, _high, _low = v
+            change = (
+                float(template[5].text.replace("%", "").replace(",", "")) / 100
+            )
+
+            return pd.DataFrame([date, _price, _open, _high, _low, change]).T
+
+        cid, data_name = loop
+        header = data_name + " Bond Yield Historical data"
+        st_date, end_date = (
+            "01/01/1970",
+            self.get_end_date(),
+        )
+        bonds_url = "https://www.investing.com/instruments/HistoricalDataAjax"
+
+        form_data = {
+            "curr_id": cid,
+            "header": header,
+            "st_date": st_date,
+            "end_date": end_date,
+            "interval_sec": "Daily",
+            "sort_col": "date",
+            "sort_ord": "DESC",
+            "action": "historical_data",
+        }
+
+        headers = {
+            "Accept": "text/plain, */*; q=0.01",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Accept-Language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+            "Connection": "keep-alive",
+            "Content-Length": "192",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Host": "www.investing.com",
+            "Origin": "https://www.investing.com",
+            "Referer": "https://www.investing.com/rates-bonds/france-1-month-bond-yield-historical-data",
+            "User-Agent": USER_AGENT,
+            "X-Requested-With": "XMLHttpRequest",
+        }
+        print("requests post")
+        res = requests.post(
+            bonds_url, verify=True, headers=headers, data=form_data
+        )
+        print("data clean")
+        page = etree.HTML(res.text)
+        tr_path = page.xpath("//tr")
+
+        col_name = [col.text for col in tr_path[0].xpath("//th")]
+        col_name = [c.replace(" %", "Percent") for c in col_name]
+        col_name = ["date" if c == "Date" else c for c in col_name]
+        data = pd.DataFrame()
+        td_path = page.xpath("//tr//td")
+        for i in range(0, len(td_path) - 6, 6):
+            tem = td_path[i : i + 6]
+            value = get_value(tem)
+            if len(value) > 0:
+                data = data.append(value)
+
+        if len(data) > 0:
+            data.columns = col_name
+            data["name"] = "{}".format(data_name)
+            # data['data_id'] = cid
+            data = data.sort_values("date")
+            data.index = range(len(data))
+
+        return data

FinMind/data/__init__.py

@@ -0,0 +1,5 @@
+from FinMind.data.finmind_api import FinMindApi
+from FinMind.data.data_loader import DataLoader
+from FinMind.data.data_subscriber import DataSubscriber
+from FinMind.data.data_subscriber import Stock
+from FinMind.data.data_subscriber import FutureAndOption