openfund-core 0.0.2__py3-none-any.whl

openfund/core/openfund_old/continuous_klines.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+import csv
+import logging
+import time
+import os
+import sys
+
+
+curPath = os.path.abspath(os.path.dirname(__file__))
+rootPath = os.path.split(curPath)[0]
+sys.path.append(rootPath)
+from datetime import datetime
+
+from binance.um_futures import UMFutures
+from binance.um_futures import UMFutures
+from binance.error import ClientError
+
+from libs.time_tools import format_timestamp
+from libs.file_tools import create_path
+from libs.prepare_env import get_api_key, get_path
+from libs import enums
+
+# 数据路径、日志路径
+data_path, log_path = get_path()
+# client
+um_futures_client = UMFutures()
+# 合同类型
+contractTypes = [type.value for type in enums.ContractType]
+# 币种类型
+# POOL = enums.CUR_SYMBOL_POOL
+POOL = [
+    "BTCUSDT_240329",
+    "BLURUSDT",
+    "DYDXUSDT",
+    "ETHBTC",
+    "ETHBUSD",
+    "ETHUSDT",
+    "ETHUSDT_231229",
+    "ETHUSDT_240329",
+    "GMTBUSD",
+    "GMTUSDT",
+    "LTCBUSD",
+    "LTCUSDT",
+    "MATICBUSD",
+    "MATICUSDT",
+    "SOLBUSD",
+    "SOLUSDT",
+]
+interval = enums.KLINE_INTERVAL_5MINUTE
+limit = 1500
+errorLimit = 100
+sleepSec = 10
+
+# 历史截止数据开关
+hisSwitch = 0
+# hisSwitch 打开的情况下，抓取数据截止时间
+hisDateTime = 1653974399999
+
+for symbol in POOL:
+    fileName = "{}/continuous_klines_{}_{}.log".format(
+        log_path, symbol, format_timestamp(datetime.now().timestamp() * 1000)
+    )
+    # print(fileName)
+    logging.basicConfig(
+        format="%(asctime)s %(name)s:%(levelname)s:%(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        level=logging.INFO,
+        filename=fileName,
+    )
+    logging.info("{} symbol 开始 ++++++++++++++++++++++++++++ ".format(symbol))
+    total = 0
+    for contractType in contractTypes:
+        latestRecords = 1
+        records = 0  # 累计记录数
+        queryTimes = 0  # 执行次数
+        nextEndTime = 0
+        errorCount = 0
+        params = {"limit": limit}
+        while latestRecords != 0:  # 循环读取，直到记录为空
+            queryTimes = queryTimes + 1
+            if nextEndTime != 0:
+                params = {"limit": limit, "endTime": nextEndTime}
+
+            logging.info(
+                "1、{}-{} 第{}次开始执行...".format(symbol, contractType, queryTimes)
+            )
+            listData = []
+            try:
+                listData = um_futures_client.continuous_klines(
+                    symbol, contractType, interval, **params
+                )
+            except KeyError as err:
+                print("KeyError:", err)
+                logging.error(err)
+                break
+            except ClientError as error:
+                logging.error(
+                    "Found error. status: {}, error code: {}, error message: {}".format(
+                        error.status_code, error.error_code, error.error_message
+                    )
+                )
+                if error.error_code == -4144 and error.status_code == 400:
+                    break
+            except Exception as e:
+                print("Error:", e)
+                logging.error(e)
+                errorCount = errorCount + 1
+                if errorCount > errorLimit:
+                    break
+                else:
+                    time.sleep(sleepSec)
+                    continue
+
+            latestRecords = len(listData)
+            records = latestRecords + records
+            logging.info("2、{}条写入文件...".format(latestRecords))
+
+            path = "{}/continuous_klines/{}/{}/".format(data_path, symbol, contractType)
+            create_path(path)  # 不存在路径进行呢创建
+
+            with open(
+                "{}continuous_klines_{}_{}_{}.csv".format(
+                    path, symbol, contractType, interval
+                ),
+                "a",
+                newline="",
+            ) as file:
+                writer = csv.writer(file)
+                # 时间戳倒序，插入文件尾部
+                writer.writerows(sorted(listData, key=lambda x: x[0], reverse=True))
+
+            # 拿到最后一条记录后,获取close时间戳,变为下一次截止时间戳
+            if latestRecords > 0:
+                nextEndTime = (
+                    listData[0][0] - 1
+                )  # -1 不和close时间戳相同，避免重新拉取重复数据
+                errorCount = 0
+                logging.info(
+                    "3、下次结束时间 {} {}".format(
+                        format_timestamp(nextEndTime), nextEndTime
+                    )
+                )
+
+                if nextEndTime <= hisDateTime and hisSwitch == 1:
+                    break
+            else:
+                logging.info("4、结束...")
+
+            total = total + records
+            logging.info("5、{} 抓取数据 {} 条记录...".format(symbol, records))
+
+    logging.info("6、{} 抓取数据 {} 条记录...".format(symbol, total))
+    logging.info("{} symbol --------------------------------- ".format(symbol))

openfund/core/openfund_old/depth.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+import csv
+import time
+import os
+import sys
+import json
+import schedule
+
+curPath = os.path.abspath(os.path.dirname(__file__))
+rootPath = os.path.split(curPath)[0]
+sys.path.append(rootPath)
+
+
+from datetime import datetime
+from binance.um_futures import UMFutures
+from binance.error import ClientError
+from libs.time_tools import format_timestamp, format_date
+from libs.file_tools import create_path
+from libs.prepare_env import get_api_key, get_path
+from libs import enums
+from libs.log_tools import Logger
+
+data_path, log_path = get_path()
+
+um_futures_client = UMFutures()
+
+POOL = enums.NEW_SYMBOL_POLL
+interval = enums.KLINE_INTERVAL_5MINUTE
+# 历史截止数据开关
+hisSwitch = 0
+# hisSwitch 打开的情况下，抓取数据截止时间
+hisDateTime = 1653974399999
+# 发现异常后累计失败次数上限，超过后退出。
+errorLimit = 100
+# 异常后暂停秒数
+sleepSec = 10
+# 下次执行周期 5min
+nextTimeMin = 5
+
+app = "depth"
+logger = Logger(app).get_log()
+
+
+# logging.basicConfig(
+#     format="%(asctime)s %(name)s:%(levelname)s:%(message)s",
+#     datefmt="%Y-%m-%d %H:%M:%S",
+#     level=logging.INFO,
+#     filename="{}/{}_{}.log".format(
+#         log_path, app, format_timestamp(datetime.now().timestamp() * 1000)
+#     ),
+# )
+def job():
+    logger.info("{} app 开始 ++++++++++++++++++++++++++++ ".format(app))
+    latestRecords = 1
+    params = {"limit": 1000}
+    for symbol in POOL:
+        queryTimes = queryTimes + 1
+        logger.info("1、{}第{}次开始执行...".format(symbol, queryTimes))
+        listData = []
+        try:
+            listData = um_futures_client.depth(symbol, **params)
+        except Exception as e:
+            logger.error(e)
+            time.sleep(sleepSec)
+            continue
+
+        latestRecords = len(listData)
+        records = latestRecords + records
+        logger.info("2、{}条写入文件...".format(latestRecords))
+
+        path = "{}/{}/".format(data_path, app)
+        create_path(path)  # 不存在路径进行呢创建
+
+        with open(
+            "{}{}_{}.txt".format(
+                path,
+                app,
+                format_date(datetime.now().timestamp() * 1000),
+            ),
+            "a",
+        ) as file:
+            file.writelines(json.dumps(listData) + "\r\n")
+
+    logger.info("3、本次 {} 抓取数据 {} 条记录...".format(app, records))
+    logger.info("{} app --------------------------------- ".format(app))
+
+
+if __name__ == "__main__":
+    schedule.every(nextTimeMin).minutes.do(job)
+    logger.info("Schedule Starting {0}min ...... ".format(nextTimeMin))
+    while True:
+        schedule.run_pending()  # 运行所有可运行的任务

openfund/core/openfund_old/historical_trades.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+import csv
+import logging
+import time
+import os
+import sys
+
+from datetime import datetime
+
+from binance.lib.utils import config_logging
+from binance.um_futures import UMFutures
+from binance.error import ClientError, Error
+
+import enums as enums
+from time_tools import format_timestamp
+from file_tools import create_path
+from prepare_env import get_api_key, get_path
+
+# config_logging(logging, logging.INFO)
+api_key, api_secret = get_api_key()
+um_futures_client = UMFutures(key=api_key)
+
+data_path, log_path = get_path()
+# POOL = ['AAVEUSDT','BTCBUSD', 'BTCDOMUSDT', 'BTCSTUSDT', 'BTCUSDT', 'BTCUSDT_231229', 'BTCUSDT_240329',
+#         'BLURUSDT', 'DYDXUSDT', 'ETHBTC', 'ETHBUSD', 'ETHUSDT', 'ETHUSDT_231229', 'ETHUSDT_240329',
+#         'GMTBUSD', 'GMTUSDT', 'LTCBUSD', 'LTCUSDT', 'MATICBUSD', 'MATICUSDT', 'SOLBUSD', 'SOLUSDT',
+#         ]
+POOL = ["GMTBUSD"]
+
+# 历史截止数据开关
+hisSwitch = 1
+# hisSwitch 打开的情况下，抓取数据截止时间
+hisDateTime = 1653974399999
+
+for symbol in POOL:
+    logging.basicConfig(
+        format="%(asctime)s %(name)s:%(levelname)s:%(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        level=logging.INFO,
+        filename="{}/{}_historical_trades_{}.log".format(
+            log_path, symbol, format_timestamp(datetime.now().timestamp() * 1000)
+        ),
+    )
+    logging.info("{} symbol 开始 ++++++++++++++++++++++++++++ ".format(symbol))
+
+    latestRecords = 1
+    records = 0  # 累计记录数
+    queryTimes = 0  # 执行次数
+    nextKey = 0
+    params = {"limit": 1000}
+
+    while latestRecords != 0:  # 循环读取，直到记录为空
+        queryTimes = queryTimes + 1
+        if nextKey != 0:
+            params = {"limit": 1000, "fromId": nextKey}
+
+        logging.info("1、{}第{}次开始执行...".format(symbol, queryTimes))
+        listData = []
+        try:
+            listData = um_futures_client.historical_trades(symbol, **params)
+        # except ClientError as error:
+        #     logging.error(
+        #         "Found error. status: {}, error code: {}, error message: {}".format(
+        #             error.status_code, error.error_code, error.error_message
+        #         )
+        #     )
+        #     time.sleep(30)
+        #     continue
+        # except ProxyError as error:
+        #     logging.error(
+        #         "Found error. error code: {}, error message: {}".format(
+        #             error, error.error_message
+        #         )
+        #     )
+        #     time.sleep(10)
+        #     continue
+        except Exception as e:
+            print("Error:", e)
+            logging.error(e)
+            time.sleep(10)
+            continue
+        # listData 结构
+        # [
+        #     {
+        #         "id": 28457,
+        #         "price": "4.00000100",
+        #         "qty": "12.00000000",
+        #         "quoteQty": "8000.00",
+        #         "time": 1499865549590,
+        #         "isBuyerMaker": true,
+        #     }
+        # ]
+        latestRecords = len(listData)
+        records = latestRecords + records
+        logging.info("2、{}条写入文件...".format(latestRecords))
+
+        path = "{}/historical_trades/{}/".format(data_path, symbol)
+
+        create_path(path)  # 不存在路径进行呢创建
+
+        with open(
+            "{}/{}_historical_trades.csv".format(path, symbol), "a", newline=""
+        ) as file:
+            writer = csv.writer(file)
+            # 时间戳倒序，插入文件尾部
+            writer.writerows(sorted(listData, key=lambda x: x["id"], reverse=True))
+
+        # 拿到最后一条记录后,获取close时间戳,变为下一次截止时间戳
+        if latestRecords > 0:
+            first = listData[0]
+            nextKey = first["id"] - 1000
+            curTime = first["time"]
+            logging.info("3、下个Key {} {}".format(format_timestamp(curTime), nextKey))
+            errorCount = 0
+            if curTime <= hisDateTime and hisSwitch == 1:
+                break
+        else:
+            logging.info("4、结束...")
+
+        time.sleep(0.01)
+
+    logging.info("5、{} 抓取数据 {} 条记录...".format(symbol, records))
+    logging.info("{} symbol --------------------------------- ".format(symbol))

openfund/core/openfund_old/index_info.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+import csv
+import time
+import os
+import sys
+import json
+import schedule
+
+curPath = os.path.abspath(os.path.dirname(__file__))
+rootPath = os.path.split(curPath)[0]
+sys.path.append(rootPath)
+
+
+from datetime import datetime
+from binance.um_futures import UMFutures
+from binance.error import ClientError
+from utils.time_tools import format_timestamp, format_date, format_date_to
+from utils.file_tools import create_path
+from utils.prepare_env import get_api_key, get_path
+from utils import enums
+from utils.log_tools import Logger
+
+data_path, log_path = get_path()
+um_futures_client = UMFutures()
+
+# 下次执行周期 5min
+nextTimeMin = 5
+
+app = "index_info"
+logger = Logger(app).get_log()
+
+
+def job():
+    logger.info("{} app 开始 ++++++++++++++++++++++++++++ ".format(app))
+    listData = []
+    try:
+        listData = um_futures_client.index_info()
+    except Exception as e:
+        print("Error:", e)
+        logger.error(e)
+
+    latestRecords = len(listData)
+
+    path = "{}/{}/".format(data_path, app)
+    create_path(path)  # 不存在路径进行呢创建
+
+    with open(
+        "{}{}_{}.txt".format(
+            path,
+            app,
+            format_date(datetime.now().timestamp() * 1000),
+        ),
+        "a",
+    ) as file:
+        file.writelines(
+            json.dumps(sorted(listData, key=lambda x: x["symbol"])) + "\r\n"
+        )
+
+    logger.info("1、本次 {} 抓取数据 {} 条记录...".format(app, records))
+    logger.info("{} app --------------------------------- ".format(app))
+
+
+if __name__ == "__main__":
+    schedule.every(nextTimeMin).minutes.do(job)
+    logger.info("Schedule Starting {0}min ...... ".format(nextTimeMin))
+    while True:
+        schedule.run_pending()  # 运行所有可运行的任务

openfund/core/openfund_old/index_price_kline.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+import csv
+import logging
+import time
+import os
+import sys
+
+curPath = os.path.abspath(os.path.dirname(__file__))
+rootPath = os.path.split(curPath)[0]
+sys.path.append(rootPath)
+
+
+from datetime import datetime
+from binance.um_futures import UMFutures
+from binance.error import ClientError
+from utils.time_tools import format_timestamp
+from utils.file_tools import create_path
+from utils.prepare_env import get_api_key, get_path
+from utils import enums
+
+data_path, log_path = get_path()
+um_futures_client = UMFutures()
+
+POOL = enums.CUR_SYMBOL_POOL
+interval = enums.KLINE_INTERVAL_5MINUTE
+# 历史截止数据开关
+hisSwitch = 0
+# hisSwitch 打开的情况下，抓取数据截止时间
+hisDateTime = 1653974399999
+# 发现异常后累计失败次数上限，超过后退出。
+errorLimit = 100
+# 异常后暂停秒数
+sleepSec = 10
+
+app = "index_price_kline"
+
+for symbol in POOL:
+    fileName = "{}/{}_{}_{}.log".format(
+        log_path, symbol, app, format_timestamp(datetime.now().timestamp() * 1000)
+    )
+    logging.basicConfig(
+        format="%(asctime)s %(name)s:%(levelname)s:%(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        level=logging.INFO,
+        filename=fileName,
+    )
+    logging.info("{}_symbol 开始 ++++++++++++++++++++++++++++ ".format(symbol))
+
+    latestRecords = 1
+    records = 0  # 累计记录数
+    queryTimes = 0  # 执行次数
+    nextEndTime = 0  # 下次结束时间
+    errorCount = 0  # 异常次数
+    params = {"limit": 1000}
+    while latestRecords != 0:  # 循环读取，直到记录为空
+        queryTimes = queryTimes + 1
+        if nextEndTime != 0:
+            params = {"limit": 1000, "endTime": nextEndTime}
+
+        logging.info("1、{}第{}次开始执行...".format(symbol, queryTimes))
+        listData = []
+        try:
+            listData = um_futures_client.index_price_klines(symbol, interval, **params)
+        except KeyError as err:
+            print("KeyError:", err)
+            logging.error(err)
+            break
+        except ClientError as error:
+            logging.error(
+                "Found error. status: {}, error code: {}, error message: {}".format(
+                    error.status_code, error.error_code, error.error_message
+                )
+            )
+            if error.error_code == -4144 and error.status_code == 400:
+                break
+        except Exception as e:
+            print("Error:", e)
+            logging.error(e)
+            errorCount = errorCount + 1
+            if errorCount > errorLimit:
+                break
+            else:
+                time.sleep(sleepSec)
+                continue
+
+        latestRecords = len(listData)
+        records = latestRecords + records
+        logging.info("2、{}条写入文件...".format(latestRecords))
+
+        path = "{}/{}/{}/".format(data_path, app, symbol)
+        create_path(path)  # 不存在路径进行呢创建
+
+        with open(
+            "{}{}_{}_{}.csv".format(path, app, symbol, interval),
+            "a",
+            newline="",
+        ) as file:
+            writer = csv.writer(file)
+            # 时间戳倒序，插入文件尾部
+            writer.writerows(sorted(listData, key=lambda x: x[0], reverse=True))
+
+        # 拿到最后一条记录后,获取close时间戳,变为下一次截止时间戳
+        if latestRecords > 0:
+            nextEndTime = listData[0][0] - 1  # -1 不和close时间戳相同，避免重新拉取重复数据
+            errorCount = 0
+            logging.info(
+                "3、下次结束时间 {} {}".format(format_timestamp(nextEndTime), nextEndTime)
+            )
+
+            if nextEndTime <= hisDateTime and hisSwitch == 1:
+                break
+        else:
+            logging.info("4、结束...")
+
+        # time.sleep(0.1)
+
+    logging.info("5、{} 抓取数据 {} 条记录...".format(symbol, records))
+    logging.info("{} symbol --------------------------------- ".format(symbol))

openfund/core/openfund_old/klines.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+import csv
+import logging
+import time
+import os
+import sys
+
+curPath = os.path.abspath(os.path.dirname(__file__))
+# print ("curPath:",curPath)
+rootPath = os.path.split(curPath)[0]
+# print ("rootPath:",rootPath)
+sys.path.append(rootPath)
+
+
+from datetime import datetime
+from binance.um_futures import UMFutures
+from binance.error import ClientError
+from utils.time_tools import format_timestamp
+from utils.file_tools import create_path
+from utils.prepare_env import get_api_key, get_path
+from utils import enums
+
+data_path, log_path = get_path()
+
+
+# config_logging(logging, logging.INFO, "../../log/future/klines.log")
+
+um_futures_client = UMFutures()
+
+POOL = ["AAVEUSDT"]
+interval = enums.KLINE_INTERVAL_5MINUTE
+# 历史截止数据开关
+hisSwitch = 0
+# hisSwitch 打开的情况下，抓取数据截止时间
+hisDateTime = 1653974399999
+
+logging.basicConfig(
+    format="%(asctime)s %(name)s:%(levelname)s:%(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    level=logging.INFO,
+    filename="{}/klines_{}.log".format(
+        log_path, format_timestamp(datetime.now().timestamp() * 1000)
+    ),
+)
+logging.info("{} symbol 开始 ++++++++++++++++++++++++++++ ".format(symbol))
+for symbol in POOL:
+    latestRecords = 1
+    records = 0  # 累计记录数
+    queryTimes = 0  # 执行次数
+    nextEndTime = 0
+    params = {"limit": 1000}
+    while latestRecords != 0:  # 循环读取，直到记录为空
+        queryTimes = queryTimes + 1
+        if nextEndTime != 0:
+            params = {"limit": 1000, "endTime": nextEndTime}
+
+        logging.info("1、{}第{}次开始执行...".format(symbol, queryTimes))
+        listData = []
+        try:
+            listData = um_futures_client.klines(symbol, interval, **params)
+        except Exception as e:
+            print("Error:", e)
+            logging.error(e)
+            time.sleep(10)
+            continue
+
+        latestRecords = len(listData)
+        records = latestRecords + records
+        logging.info("2、{}条写入文件...".format(latestRecords))
+
+        path = "{}/klines/{}/".format(data_path, symbol)
+        create_path(path)  # 不存在路径进行呢创建
+
+        with open("{}klines_{}.csv".format(path, interval), "a", newline="") as file:
+            writer = csv.writer(file)
+            # 时间戳倒序，插入文件尾部
+            writer.writerows(sorted(listData, key=lambda x: x[0], reverse=True))
+
+        # 拿到最后一条记录后,获取close时间戳,变为下一次截止时间戳
+        if latestRecords > 0:
+            nextEndTime = listData[0][0] - 1  # -1 不和close时间戳相同，避免重新拉取重复数据
+            errorCount = 0
+            logging.info(
+                "3、下次结束时间 {} {}".format(format_timestamp(nextEndTime), nextEndTime)
+            )
+
+            if nextEndTime <= hisDateTime and hisSwitch == 1:
+                break
+        else:
+            logging.info("4、结束...")
+
+        # time.sleep(0.1)
+
+    logging.info("5、{} 抓取数据 {} 条记录...".format(symbol, records))
+    logging.info("{} symbol --------------------------------- ".format(symbol))