openfund-core 0.0.3__py3-none-any.whl → 1.0.1__py3-none-any.whl

openfund/core/base_collector.py

@@ -1,72 +0,0 @@
-from __future__ import annotations
-
-import logging
-import csv
-
-from abc import abstractmethod
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-from apscheduler.job import Job
-
-from openfund.core.pyopenfund import Openfund
-from openfund.core.factory import Factory
-
-
-logger = logging.getLogger(__name__)
-
-
-class Collector:
-    def __init__(self, openfund: Openfund | None = None) -> None:
-        self._openfund: Openfund = openfund
-        if self._openfund is None:
-            self._openfund = Factory.create_openfund()
-        self._job: Job = None
-
-    @property
-    def openfund(self) -> Openfund:
-        return self._openfund
-
-    @abstractmethod
-    def collect(self) -> None:
-        raise NotImplementedError()
-
-    @abstractmethod
-    def start(self) -> int:
-        raise NotImplementedError()
-        # from apscheduler.schedulers.background import BlockingScheduler
-
-        # apSchedule = BlockingScheduler()
-        # self.openfund.scheduler.add_job(
-        #     func=self.collect, trigger="interval", minutes=5,seconds=5 id="um_futures_collector"
-        # )
-
-    def stop(self) -> int:
-        if self._job is not None:
-            self._job.remove()
-        logger.debug(f"{self._job.name} is stop .")
-        return 0
-
-    def pause(self) -> int:
-        if self._job is not None:
-            self._job.pause()
-        logger.debug(f"{self._job.name} is pause .")
-        return 0
-
-    def resume(self) -> int:
-        if self._job is not None:
-            self._job.resume()
-        logger.debug(f"{self._job.name} is resume .")
-        return 0
-
-    def _write_to_csv(self, file: Path, listData: list) -> None:
-
-        # 如果路径不存在，创建路径
-        file.parent.mkdir(parents=True, exist_ok=True)
-
-        with open(file, "a", newline="") as file:
-            writer = csv.writer(file)
-            # 时间戳倒序，插入文件尾部
-            writer.writerows(sorted(listData, key=lambda x: x[0], reverse=True))
-
-        logger.debug("2、{}条写入{}文件...".format(len(listData), file))

openfund/core/base_tool.py

@@ -1,58 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from pathlib import Path
-
-from typing import TYPE_CHECKING
-
-from poetry.utils.authenticator import Authenticator
-from binance.spot import Spot as Client
-from binance.um_futures import UMFutures as UMClient
-
-from openfund.core.pyopenfund import Openfund
-from openfund.core.factory import Factory
-
-
-logger = logging.getLogger(__name__)
-
-
-class Tool:
-    def __init__(
-        self, openfund: Openfund | None = None, toolname: str = "binance"
-    ) -> None:
-        self._openfund: Openfund = openfund
-        if self._openfund is None:
-            self._openfund = Factory.create_openfund()
-
-        self._toolname = toolname
-        self._password_manager = Authenticator(
-            self._openfund._poetry.config
-        )._password_manager
-
-        self._client = None
-        self._umclient = None
-
-    @property
-    def api_key(self) -> str:
-        return self._password_manager.get_http_auth(self._toolname).get("username")
-
-    @property
-    def apk_secret(self) -> str:
-        return self._password_manager.get_http_auth(self._toolname).get("password")
-
-    @property
-    def openfund(self) -> Openfund:
-        return self._openfund
-
-    @property
-    def client(self) -> Client:
-        if self._client is None:
-            self._client = Client(self.api_key, self.apk_secret)
-        return self._client
-
-    @property
-    def umclient(self) -> UMClient:
-        if self._umclient is None:
-            self._umclient = UMClient(self.api_key, self.apk_secret)
-        return self._umclient

openfund/core/factory.py

@@ -1,97 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-from poetry.factory import Factory as BaseFactory
-from openfund.core.pyopenfund import Openfund
-
-
-if TYPE_CHECKING:
-    from poetry.poetry import Poetry
-
-
-logger = logging.getLogger(__name__)
-_APP_NAME = "pyopenfund"
-
-
-class Factory(BaseFactory):
-    _openfund: Openfund = None
-
-    def __init__(self) -> None:
-        super().__init__()
-        self._init_log()
-
-    def create_poetry(
-        self,
-        cwd: Path | None = None,
-        with_groups: bool = True,
-    ) -> Poetry:
-        poetry = super().create_poetry(cwd=cwd, with_groups=with_groups)
-        return poetry
-
-    @classmethod
-    def create_openfund(
-        cls,
-        cwd: Path | None = None,
-        with_groups: bool = True,
-    ) -> Openfund:
-        if cls._openfund is not None:
-            return cls._openfund
-
-        if cwd is None:
-            cwd = Path.cwd()
-
-        poetry = Factory().create_poetry(cwd=cwd, with_groups=with_groups)
-        cls._openfund = Openfund(poetry)
-        cls._openfund.scheduler.start()
-        return cls._openfund
-
-    def _init_log(self):
-        from openfund.core.pyopenfund import user_log_path
-
-        log_file = user_log_path(
-            _APP_NAME, appauthor=False, ensure_exists=True
-        ).resolve()
-        log_file = (
-            user_log_path(_APP_NAME, appauthor=False, ensure_exists=True)
-            .joinpath("openfund-core.log")
-            .resolve()
-        )
-        fileHandler = FileHandler(log_file)
-        fileHandler.setFormatter(FileFormatter())
-        console_handler = logging.StreamHandler()
-        console_handler.setFormatter(FileFormatter())
-        logging.basicConfig(
-            level=logging.DEBUG,
-            handlers=[console_handler, fileHandler],
-        )
-
-
-from logging.handlers import TimedRotatingFileHandler
-
-
-class FileHandler(TimedRotatingFileHandler):
-    def __init__(
-        self,
-        filename,
-        when="midnight",
-        interval=1,
-        backupCount=7,
-        encoding=None,
-        delay=False,
-        utc=False,
-    ) -> None:
-        super().__init__(filename, when, interval, backupCount, encoding, delay, utc)
-
-
-class FileFormatter(logging.Formatter):
-
-    _format = "%(asctime)s - %(process)d | %(threadName)s | %(module)s.%(funcName)s:%(lineno)d - %(levelname)s -%(message)s"
-
-    _datefmt = "%Y-%m-%d-%H:%M:%S"  # 时间
-
-    def __init__(self, fmt=_format, datefmt=_datefmt, style="%") -> None:
-        super().__init__(fmt, datefmt, style)

openfund/core/openfund_old/continuous_klines.py

@@ -1,153 +0,0 @@
-#!/usr/bin/env python
-import csv
-import logging
-import time
-import os
-import sys
-
-
-curPath = os.path.abspath(os.path.dirname(__file__))
-rootPath = os.path.split(curPath)[0]
-sys.path.append(rootPath)
-from datetime import datetime
-
-from binance.um_futures import UMFutures
-from binance.um_futures import UMFutures
-from binance.error import ClientError
-
-from libs.time_tools import format_timestamp
-from libs.file_tools import create_path
-from libs.prepare_env import get_api_key, get_path
-from libs import enums
-
-# 数据路径、日志路径
-data_path, log_path = get_path()
-# client
-um_futures_client = UMFutures()
-# 合同类型
-contractTypes = [type.value for type in enums.ContractType]
-# 币种类型
-# POOL = enums.CUR_SYMBOL_POOL
-POOL = [
-    "BTCUSDT_240329",
-    "BLURUSDT",
-    "DYDXUSDT",
-    "ETHBTC",
-    "ETHBUSD",
-    "ETHUSDT",
-    "ETHUSDT_231229",
-    "ETHUSDT_240329",
-    "GMTBUSD",
-    "GMTUSDT",
-    "LTCBUSD",
-    "LTCUSDT",
-    "MATICBUSD",
-    "MATICUSDT",
-    "SOLBUSD",
-    "SOLUSDT",
-]
-interval = enums.KLINE_INTERVAL_5MINUTE
-limit = 1500
-errorLimit = 100
-sleepSec = 10
-
-# 历史截止数据开关
-hisSwitch = 0
-# hisSwitch 打开的情况下，抓取数据截止时间
-hisDateTime = 1653974399999
-
-for symbol in POOL:
-    fileName = "{}/continuous_klines_{}_{}.log".format(
-        log_path, symbol, format_timestamp(datetime.now().timestamp() * 1000)
-    )
-    # print(fileName)
-    logging.basicConfig(
-        format="%(asctime)s %(name)s:%(levelname)s:%(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S",
-        level=logging.INFO,
-        filename=fileName,
-    )
-    logging.info("{} symbol 开始 ++++++++++++++++++++++++++++ ".format(symbol))
-    total = 0
-    for contractType in contractTypes:
-        latestRecords = 1
-        records = 0  # 累计记录数
-        queryTimes = 0  # 执行次数
-        nextEndTime = 0
-        errorCount = 0
-        params = {"limit": limit}
-        while latestRecords != 0:  # 循环读取，直到记录为空
-            queryTimes = queryTimes + 1
-            if nextEndTime != 0:
-                params = {"limit": limit, "endTime": nextEndTime}
-
-            logging.info(
-                "1、{}-{} 第{}次开始执行...".format(symbol, contractType, queryTimes)
-            )
-            listData = []
-            try:
-                listData = um_futures_client.continuous_klines(
-                    symbol, contractType, interval, **params
-                )
-            except KeyError as err:
-                print("KeyError:", err)
-                logging.error(err)
-                break
-            except ClientError as error:
-                logging.error(
-                    "Found error. status: {}, error code: {}, error message: {}".format(
-                        error.status_code, error.error_code, error.error_message
-                    )
-                )
-                if error.error_code == -4144 and error.status_code == 400:
-                    break
-            except Exception as e:
-                print("Error:", e)
-                logging.error(e)
-                errorCount = errorCount + 1
-                if errorCount > errorLimit:
-                    break
-                else:
-                    time.sleep(sleepSec)
-                    continue
-
-            latestRecords = len(listData)
-            records = latestRecords + records
-            logging.info("2、{}条写入文件...".format(latestRecords))
-
-            path = "{}/continuous_klines/{}/{}/".format(data_path, symbol, contractType)
-            create_path(path)  # 不存在路径进行呢创建
-
-            with open(
-                "{}continuous_klines_{}_{}_{}.csv".format(
-                    path, symbol, contractType, interval
-                ),
-                "a",
-                newline="",
-            ) as file:
-                writer = csv.writer(file)
-                # 时间戳倒序，插入文件尾部
-                writer.writerows(sorted(listData, key=lambda x: x[0], reverse=True))
-
-            # 拿到最后一条记录后,获取close时间戳,变为下一次截止时间戳
-            if latestRecords > 0:
-                nextEndTime = (
-                    listData[0][0] - 1
-                )  # -1 不和close时间戳相同，避免重新拉取重复数据
-                errorCount = 0
-                logging.info(
-                    "3、下次结束时间 {} {}".format(
-                        format_timestamp(nextEndTime), nextEndTime
-                    )
-                )
-
-                if nextEndTime <= hisDateTime and hisSwitch == 1:
-                    break
-            else:
-                logging.info("4、结束...")
-
-            total = total + records
-            logging.info("5、{} 抓取数据 {} 条记录...".format(symbol, records))
-
-    logging.info("6、{} 抓取数据 {} 条记录...".format(symbol, total))
-    logging.info("{} symbol --------------------------------- ".format(symbol))

openfund/core/openfund_old/depth.py

@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-import csv
-import time
-import os
-import sys
-import json
-import schedule
-
-curPath = os.path.abspath(os.path.dirname(__file__))
-rootPath = os.path.split(curPath)[0]
-sys.path.append(rootPath)
-
-
-from datetime import datetime
-from binance.um_futures import UMFutures
-from binance.error import ClientError
-from libs.time_tools import format_timestamp, format_date
-from libs.file_tools import create_path
-from libs.prepare_env import get_api_key, get_path
-from libs import enums
-from libs.log_tools import Logger
-
-data_path, log_path = get_path()
-
-um_futures_client = UMFutures()
-
-POOL = enums.NEW_SYMBOL_POLL
-interval = enums.KLINE_INTERVAL_5MINUTE
-# 历史截止数据开关
-hisSwitch = 0
-# hisSwitch 打开的情况下，抓取数据截止时间
-hisDateTime = 1653974399999
-# 发现异常后累计失败次数上限，超过后退出。
-errorLimit = 100
-# 异常后暂停秒数
-sleepSec = 10
-# 下次执行周期 5min
-nextTimeMin = 5
-
-app = "depth"
-logger = Logger(app).get_log()
-
-
-# logging.basicConfig(
-#     format="%(asctime)s %(name)s:%(levelname)s:%(message)s",
-#     datefmt="%Y-%m-%d %H:%M:%S",
-#     level=logging.INFO,
-#     filename="{}/{}_{}.log".format(
-#         log_path, app, format_timestamp(datetime.now().timestamp() * 1000)
-#     ),
-# )
-def job():
-    logger.info("{} app 开始 ++++++++++++++++++++++++++++ ".format(app))
-    latestRecords = 1
-    params = {"limit": 1000}
-    for symbol in POOL:
-        queryTimes = queryTimes + 1
-        logger.info("1、{}第{}次开始执行...".format(symbol, queryTimes))
-        listData = []
-        try:
-            listData = um_futures_client.depth(symbol, **params)
-        except Exception as e:
-            logger.error(e)
-            time.sleep(sleepSec)
-            continue
-
-        latestRecords = len(listData)
-        records = latestRecords + records
-        logger.info("2、{}条写入文件...".format(latestRecords))
-
-        path = "{}/{}/".format(data_path, app)
-        create_path(path)  # 不存在路径进行呢创建
-
-        with open(
-            "{}{}_{}.txt".format(
-                path,
-                app,
-                format_date(datetime.now().timestamp() * 1000),
-            ),
-            "a",
-        ) as file:
-            file.writelines(json.dumps(listData) + "\r\n")
-
-    logger.info("3、本次 {} 抓取数据 {} 条记录...".format(app, records))
-    logger.info("{} app --------------------------------- ".format(app))
-
-
-if __name__ == "__main__":
-    schedule.every(nextTimeMin).minutes.do(job)
-    logger.info("Schedule Starting {0}min ...... ".format(nextTimeMin))
-    while True:
-        schedule.run_pending()  # 运行所有可运行的任务

openfund/core/openfund_old/historical_trades.py

@@ -1,123 +0,0 @@
-#!/usr/bin/env python
-import csv
-import logging
-import time
-import os
-import sys
-
-from datetime import datetime
-
-from binance.lib.utils import config_logging
-from binance.um_futures import UMFutures
-from binance.error import ClientError, Error
-
-import enums as enums
-from time_tools import format_timestamp
-from file_tools import create_path
-from prepare_env import get_api_key, get_path
-
-# config_logging(logging, logging.INFO)
-api_key, api_secret = get_api_key()
-um_futures_client = UMFutures(key=api_key)
-
-data_path, log_path = get_path()
-# POOL = ['AAVEUSDT','BTCBUSD', 'BTCDOMUSDT', 'BTCSTUSDT', 'BTCUSDT', 'BTCUSDT_231229', 'BTCUSDT_240329',
-#         'BLURUSDT', 'DYDXUSDT', 'ETHBTC', 'ETHBUSD', 'ETHUSDT', 'ETHUSDT_231229', 'ETHUSDT_240329',
-#         'GMTBUSD', 'GMTUSDT', 'LTCBUSD', 'LTCUSDT', 'MATICBUSD', 'MATICUSDT', 'SOLBUSD', 'SOLUSDT',
-#         ]
-POOL = ["GMTBUSD"]
-
-# 历史截止数据开关
-hisSwitch = 1
-# hisSwitch 打开的情况下，抓取数据截止时间
-hisDateTime = 1653974399999
-
-for symbol in POOL:
-    logging.basicConfig(
-        format="%(asctime)s %(name)s:%(levelname)s:%(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S",
-        level=logging.INFO,
-        filename="{}/{}_historical_trades_{}.log".format(
-            log_path, symbol, format_timestamp(datetime.now().timestamp() * 1000)
-        ),
-    )
-    logging.info("{} symbol 开始 ++++++++++++++++++++++++++++ ".format(symbol))
-
-    latestRecords = 1
-    records = 0  # 累计记录数
-    queryTimes = 0  # 执行次数
-    nextKey = 0
-    params = {"limit": 1000}
-
-    while latestRecords != 0:  # 循环读取，直到记录为空
-        queryTimes = queryTimes + 1
-        if nextKey != 0:
-            params = {"limit": 1000, "fromId": nextKey}
-
-        logging.info("1、{}第{}次开始执行...".format(symbol, queryTimes))
-        listData = []
-        try:
-            listData = um_futures_client.historical_trades(symbol, **params)
-        # except ClientError as error:
-        #     logging.error(
-        #         "Found error. status: {}, error code: {}, error message: {}".format(
-        #             error.status_code, error.error_code, error.error_message
-        #         )
-        #     )
-        #     time.sleep(30)
-        #     continue
-        # except ProxyError as error:
-        #     logging.error(
-        #         "Found error. error code: {}, error message: {}".format(
-        #             error, error.error_message
-        #         )
-        #     )
-        #     time.sleep(10)
-        #     continue
-        except Exception as e:
-            print("Error:", e)
-            logging.error(e)
-            time.sleep(10)
-            continue
-        # listData 结构
-        # [
-        #     {
-        #         "id": 28457,
-        #         "price": "4.00000100",
-        #         "qty": "12.00000000",
-        #         "quoteQty": "8000.00",
-        #         "time": 1499865549590,
-        #         "isBuyerMaker": true,
-        #     }
-        # ]
-        latestRecords = len(listData)
-        records = latestRecords + records
-        logging.info("2、{}条写入文件...".format(latestRecords))
-
-        path = "{}/historical_trades/{}/".format(data_path, symbol)
-
-        create_path(path)  # 不存在路径进行呢创建
-
-        with open(
-            "{}/{}_historical_trades.csv".format(path, symbol), "a", newline=""
-        ) as file:
-            writer = csv.writer(file)
-            # 时间戳倒序，插入文件尾部
-            writer.writerows(sorted(listData, key=lambda x: x["id"], reverse=True))
-
-        # 拿到最后一条记录后,获取close时间戳,变为下一次截止时间戳
-        if latestRecords > 0:
-            first = listData[0]
-            nextKey = first["id"] - 1000
-            curTime = first["time"]
-            logging.info("3、下个Key {} {}".format(format_timestamp(curTime), nextKey))
-            errorCount = 0
-            if curTime <= hisDateTime and hisSwitch == 1:
-                break
-        else:
-            logging.info("4、结束...")
-
-        time.sleep(0.01)
-
-    logging.info("5、{} 抓取数据 {} 条记录...".format(symbol, records))
-    logging.info("{} symbol --------------------------------- ".format(symbol))

openfund/core/openfund_old/index_info.py

@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-import csv
-import time
-import os
-import sys
-import json
-import schedule
-
-curPath = os.path.abspath(os.path.dirname(__file__))
-rootPath = os.path.split(curPath)[0]
-sys.path.append(rootPath)
-
-
-from datetime import datetime
-from binance.um_futures import UMFutures
-from binance.error import ClientError
-from utils.time_tools import format_timestamp, format_date, format_date_to
-from utils.file_tools import create_path
-from utils.prepare_env import get_api_key, get_path
-from utils import enums
-from utils.log_tools import Logger
-
-data_path, log_path = get_path()
-um_futures_client = UMFutures()
-
-# 下次执行周期 5min
-nextTimeMin = 5
-
-app = "index_info"
-logger = Logger(app).get_log()
-
-
-def job():
-    logger.info("{} app 开始 ++++++++++++++++++++++++++++ ".format(app))
-    listData = []
-    try:
-        listData = um_futures_client.index_info()
-    except Exception as e:
-        print("Error:", e)
-        logger.error(e)
-
-    latestRecords = len(listData)
-
-    path = "{}/{}/".format(data_path, app)
-    create_path(path)  # 不存在路径进行呢创建
-
-    with open(
-        "{}{}_{}.txt".format(
-            path,
-            app,
-            format_date(datetime.now().timestamp() * 1000),
-        ),
-        "a",
-    ) as file:
-        file.writelines(
-            json.dumps(sorted(listData, key=lambda x: x["symbol"])) + "\r\n"
-        )
-
-    logger.info("1、本次 {} 抓取数据 {} 条记录...".format(app, records))
-    logger.info("{} app --------------------------------- ".format(app))
-
-
-if __name__ == "__main__":
-    schedule.every(nextTimeMin).minutes.do(job)
-    logger.info("Schedule Starting {0}min ...... ".format(nextTimeMin))
-    while True:
-        schedule.run_pending()  # 运行所有可运行的任务