PipeGraphPy 2.0.6__py3-none-win_amd64.whl

PipeGraphPy/db/utils.py

@@ -0,0 +1,106 @@
+import json
+import traceback
+from datetime import datetime, timedelta
+from PipeGraphPy.db.models import GraphsTB, NodesTB, ObjectsTB
+from dbpoolpy import connect_db
+from PipeGraphPy.logger import log
+from PipeGraphPy.constants import BIZ_TYPE, STATUS, DB
+
+
+def update_node_params(node_ids, key, value, value_key="value"):
+    """更新节点的params字段信息
+    parametes:
+        node_ids: list or int 要更新节点的id
+        key: str 要更新的键
+        value: str 要更新的值
+    """
+    if isinstance(node_ids, int):
+        node_ids = [node_ids]
+    elif isinstance(node_ids, list):
+        node_ids = node_ids
+    else:
+        raise Exception("update_node_params函数node_ids字段传值类型不合法")
+    for node_id in node_ids:
+        node_info = NodesTB.find_one(id=node_id)
+        node_params = json.loads(node_info["params"])
+        for i in node_params:
+            if i["key"] == key:
+                i[value_key] = value
+                break
+        NodesTB.set(params=json.dumps(node_params)).where(id=node_id)
+
+
+def insert_batch(dbserver, table, datas, num=2000):
+    try:
+        cnt = 0
+        with connect_db(dbserver) as db:
+            while len(datas) > cnt:
+                ceil_list = datas[cnt : cnt + num]
+                db.insert(table).many(ceil_list).execute()
+                cnt += num
+    except Exception:
+        log.error(traceback.format_exc())
+
+
+def df_to_db(dbserver, table, df):
+    if df.empty:
+        return
+    datas = df.to_dict(orient="records")
+    insert_batch(dbserver, table, datas)
+
+
+def get_object_info(object_id):
+    """获取业务信息
+    参数：
+        object_id：业务id
+    """
+    try:
+        object_info = ObjectsTB.find_one(id=object_id)
+        if object_info:
+            if object_info.get("ext_json"):
+                try:
+                    ext_params = json.loads(object_info["ext_json"])
+                    object_info.update(ext_params)
+                except:
+                    ext_params = {}
+        else:
+            object_info = {"id": object_id}
+        return object_info
+    except Exception as e:
+        print(traceback.format_exc())
+        return {"id": object_id}
+
+def graphs_wait_to_run(graph_ids):
+    """模型进入训练队列"""
+    if graph_ids:
+        running_ids = GraphsTB.map("id").where(
+                status=("in", [STATUS.RUNNING, STATUS.WAITRUN]),
+                id=("in",graph_ids))
+        if running_ids:
+            graph_ids = list(set(graph_ids) - set(running_ids))
+        if graph_ids:
+            now = (datetime.utcnow()+timedelta(hours=8)).strftime("%Y-%m-%d %H:%M:%S")
+            GraphsTB.set(
+                status=STATUS.WAITRUN,
+                wait_run_time=now,
+                task_id=''
+            ).where(id=("in", graph_ids))
+
+def graphs_wait_to_predict(graph_ids, clock="12", is_pub=False, **kwargs):
+    """模型进入预测队列"""
+    params = {"clock": clock, "is_pub": is_pub}
+    if kwargs:
+        params.update(kwargs)
+    if graph_ids:
+        predicting_ids = GraphsTB.map("id").where(
+                p_status=("in", [STATUS.RUNNING, STATUS.WAITRUN]),
+                id=("in",graph_ids))
+        if predicting_ids:
+            graph_ids = list(set(graph_ids) - set(predicting_ids))
+        if graph_ids:
+            now = (datetime.utcnow()+timedelta(hours=8)).strftime("%Y-%m-%d %H:%M:%S")
+            GraphsTB.set(
+                p_status=STATUS.WAITRUN,
+                predict_params=json.dumps(params),
+                wait_predict_time=now,
+            ).where(id=("in", graph_ids))

PipeGraphPy/decorators.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+class _Missing(object):
+    def __repr__(self):
+        return "no value"
+
+    def __reduce__(self):
+        return "_missing"
+
+_missing = _Missing()
+
+class cached_property(property):
+    """将函数转换为惰性属性的装饰器。包装的函数在第一次使用结果时被调用，
+       然后在下次访问该值时使用该计算缓存的结果:
+
+        class Foo(object):
+
+            @cached_property
+            def foo(self):
+                # calculate something important here
+                return 42
+    """
+
+    def __init__(self, func, name=None, doc=None):
+        self.__name__ = name or func.__name__
+        self.__module__ = func.__module__
+        self.__doc__ = doc or func.__doc__
+        self.func = func
+
+    def __set__(self, obj, value):
+        obj.__dict__[self.__name__] = value
+
+    def __get__(self, obj, type=None):
+        if obj is None:
+            return self
+        value = obj.__dict__.get(self.__name__, _missing)
+        if value is _missing:
+            value = self.func(obj)
+            obj.__dict__[self.__name__] = value
+        return value

PipeGraphPy/logger.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import re
+import logging
+import logging, logging.config
+from logging.handlers import TimedRotatingFileHandler
+from PipeGraphPy.config import settings
+from PipeGraphPy.db.models import PredictRecordTB, OnlineGraphsPredictRecordTB, RunRecordTB, BacktestRecordTB, EvaluateRecordTB, OnlineGraphsEvaluateRecordTB
+
+class RecordHandler(logging.Handler, object):
+    """
+    自定义日志handler
+    """
+    def __init__(self, name, other_attr=None, **kwargs):
+        logging.Handler.__init__(self)
+        self.kw = kwargs
+
+    def emit(self, record):
+        """
+        emit函数为自定义handler类时必重写的函数，这里可以根据需要对日志消息做一些处理
+        """
+        try:
+            msg = self.format(record)
+            if settings.SDK_SHOW_LOG:
+                print(msg)
+            if self.kw.get("plog_record_id"):
+                PredictRecordTB.add_log(id=self.kw["plog_record_id"], msg=msg)
+            if self.kw.get("rlog_record_id"):
+                RunRecordTB.add_log(id=self.kw["rlog_record_id"], msg=msg)
+            if self.kw.get("online_plog_record_id"):
+                OnlineGraphsPredictRecordTB.add_log(id=self.kw["online_plog_record_id"], msg=msg)
+            if self.kw.get("backtest_record_id"):
+                BacktestRecordTB.add_log(id=self.kw["backtest_record_id"], msg=msg)
+            if self.kw.get("evaluate_record_id"):
+                EvaluateRecordTB.add_log(id=self.kw["evaluate_record_id"], msg=msg)
+            if self.kw.get("online_evaluate_record_id"):
+                OnlineGraphsEvaluateRecordTB.add_log(id=self.kw["online_evaluate_record_id"], msg=msg)
+        except Exception:
+            self.handleError(record)
+
+
+def get_pgp_logger(log_name='', logfile='', level=logging.INFO,
+        backup_count=7, handlers=['console'], **kwargs):
+    # 创建logger对象。传入logger名字
+    logger = logging.getLogger(log_name)
+    # 设置日志记录等级
+    logger.setLevel(level)
+    if 'file' in handlers:
+        # interval 滚动周期，
+        # when="MIDNIGHT", interval=1 表示每天0点为更新点，每天生成一个文件
+        # backupCount  表示日志保存个数
+        file_handler = TimedRotatingFileHandler(
+            filename=logfile, when="MIDNIGHT", interval=1, backupCount=backup_count
+        )
+        # filename="mylog" suffix设置，会生成文件名为mylog.2020-02-25.log
+        file_handler.suffix = "%Y-%m-%d.log"
+        # extMatch是编译好正则表达式，用于匹配日志文件名后缀
+        # 需要注意的是suffix和extMatch一定要匹配的上，如果不匹配，过期日志不会被删除。
+        file_handler.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}.log$")
+        # 定义日志输出格式
+        file_handler.setFormatter(
+            logging.Formatter(
+                "[%(asctime)s] [%(process)d] [%(levelname)s] - %(message)s"
+            )
+        )
+        logger.addHandler(file_handler)
+    if 'console' in handlers:
+        logger.addHandler(logging.StreamHandler())
+    if 'record' in handlers:
+        logger.addHandler(RecordHandler(log_name, **kwargs))
+    return logger
+
+
+def log_dec(level):
+    def wrapper(func):
+        def _(cls, *msg, **extra):
+            try:
+                msg = '\n'.join([str(i) for i in msg])
+                if not hasattr(cls, "logger") or cls.logger is None:
+                    cls.logger = get_pgp_logger(
+                        logfile=extra.get("pgp_logfile") or "",
+                        log_name=extra.get("pgp_log_name") or "pgp",
+                        level=extra.get("pgp_level") or logging.INFO,
+                        backup_count=extra.get("pgp_backup_count") or 7,
+                        handlers=extra.get("pgp_handlers") or ["console"],
+                        **extra
+                    )
+                f = getattr(cls.logger, level)
+                f(msg)
+            except:
+                pass
+        return _
+    return wrapper
+
+
+def get_logger(folder="run"):
+    # 不需要mq消息记录日志
+    return logging.getLogger(folder)
+
+
+class LogBase(object):
+    __function__ = None
+
+    @classmethod
+    def init(cls, pgp_handlers, **extra):
+        cls.logger = get_pgp_logger(
+            logfile=extra.get("pgp_logfile") or "",
+            log_name=extra.get("pgp_log_name") or "pgp",
+            level=extra.get("pgp_level") or logging.INFO,
+            backup_count=extra.get("pgp_backup_count") or 7,
+            handlers=pgp_handlers,
+            **extra
+        )
+
+    @classmethod
+    @log_dec("info")
+    def info(cls, msg, **extra):
+        pass
+
+    @classmethod
+    @log_dec("error")
+    def error(cls, msg, **extra):
+        pass
+
+    @classmethod
+    @log_dec("warning")
+    def warning(cls, msg, **extra):
+        pass
+
+    @classmethod
+    @log_dec("warn")
+    def warn(cls, msg, **extra):
+        pass
+
+    @classmethod
+    @log_dec("debug")
+    def debug(cls, msg, **extra):
+        pass
+
+
+class plog(LogBase):
+    """预测使用的log"""
+
+    __function__ = "predict"
+    logger = None
+
+    @classmethod
+    def log(cls, level, msg, **kwargs):
+        if hasattr(cls, level):
+            f = getattr(cls, level)
+            f(msg, **kwargs)
+
+
+class rlog(LogBase):
+    """运行使用的log"""
+
+    __function__ = "run"
+    logger = None
+
+
+class log(LogBase):
+    """一般log"""
+    logger = None
+
+
+
+if __name__ == "__main__":
+    test_log = get_pgp_logger("test")
+    test_log.info('test')