PipeGraphPy 2.0.3__tar.gz → 2.0.5__tar.gz

pipegraphpy-2.0.5/.gitignore

@@ -0,0 +1,26 @@
+*.pyc
+#*.txt
+# *.csv
+imlf/json_files/*
+checkpoint
+mymodel.tflearn.*
+*.egg-info
+dist
+*.log
+.vscode/*
+.cache
+build/*
+json_files/*
+algo_data/*
+.pytest_cache/
+.scannerwork/
+sonar-report/
+.coverage
+__pycache__/
+structured_data_regressor/
+model_files/*
+logs/*
+*.pid
+demo/*
+tests/*
+competition/*

pipegraphpy-2.0.5/.pypi_token

@@ -0,0 +1,3 @@
+UploadAllProj
+pypi-AgEIcHlwaS5vcmcCJGE0MDhjYWE4LWRmNmQtNGYzMC1iMGI5LThlMTY4OWRjNDRhZQACKlszLCJlNTlhODZmMi1hMmUwLTRmM2QtOTEzMS0wNzUzMmUyOTZiY2QiXQAABiD3Wap-15uU01YIt5_XMMn2v_8bwojhghDi1Q446SDqwg
+

{pipegraphpy-2.0.3/src/PipeGraphPy.egg-info → pipegraphpy-2.0.5}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: PipeGraphPy
-Version: 2.0.3
+Version: 2.0.5
 Summary: 核心算法框架
 Home-page: https://gitee.com/zsq_tee/pipe-graph-py
-Author: zhengshuiqing
-Author-email: shuiqingzheng@126.com
+Author: liujm
+Author-email: 792079829@qq.com
 License: Private
 Project-URL: Documentation, https://gitee.com/zsq_tee/pipe-graph-py
 Project-URL: Source, https://gitee.com/zsq_tee/pipe-graph-py

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/README.md

@@ -87,7 +87,7 @@ class ImportExample():
 
     def __init__(self, **kw):
         self.params = kw
-        self.farm_info = kw.get("objects_info", {})
+        self.farm_info = kw.get("object_info", {})
 
     def run(self):
         df = pd.DataFrame(
@@ -371,7 +371,7 @@ class SqlFilter():
     __version__ = '0.0.2'
     def __init__(self, **kw):
         self.params = kw
-        self.farm_info = kw.get('objects_info')
+        self.farm_info = kw.get('object_info')
 
     def _sqlfilter(self, sql, X, y=None):
         X, y = X, pd.DataFrame() if y is None else y

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/setup.cfg

@@ -1,8 +1,8 @@
 [metadata]
 name = PipeGraphPy
 version = attr: PipeGraphPy.__version__
-author = zhengshuiqing
-author_email = shuiqingzheng@126.com
+author = liujm
+author_email = 792079829@qq.com
 url = https://gitee.com/zsq_tee/pipe-graph-py
 description = A Python library of core algorithms for weather group forecasting
 long_description = file: README.rst

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/__init__.py

@@ -1,6 +1,6 @@
 # coding:utf-8
 
-__version__ = "2.0.3"
+__version__ = "2.0.5"
 
 from PipeGraphPy.core.graph import Graph, graph_predict, graph_evaluate, online_graph_evaluate, online_graph_predict, graph_run, graph_backtest
 from PipeGraphPy.core.module import Module

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/config/default_settings.py

@@ -30,7 +30,7 @@ BACKTEST_RESULT_SAVE_PATH = os.environ.get("PGP_BACKTEST_RESULT_SAVE_PATH", '')
 RUN_RESULT_SAVE_PATH = os.environ.get("PGP_RUN_RESULT_SAVE_PATH", '')
 EVALUATE_RESULT_SAVE_PATH = os.environ.get("PGP_EVALUATE_RESULT_SAVE_PATH", '')
 TEMP_SAVE_PATH = os.environ.get("PGP_TEMP_SAVE_PATH", "")
-CUSTOM_CACHE_PATH = os.environ.get("PGP_CUSTOM_CACHE_PATH", '')
+MODULES_CACHE_PATH = os.environ.get("PGP_MODULES_CACHE_PATH", '')
 REDIS_HOST = os.environ.get("REDIS_HOST", '39.105.185.60')
 REDIS_PORT = int(os.environ.get("REDIS_PORT", '6371'))
 PGP_DB_HOST = os.environ.get("PGP_DB_HOST", '39.105.185.60')
@@ -68,15 +68,15 @@ if not EVALUATE_RESULT_SAVE_PATH:
     EVALUATE_RESULT_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "evaluate_result")
 if not TEMP_SAVE_PATH:
     TEMP_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "temp")
-if not CUSTOM_CACHE_PATH:
-    CUSTOM_CACHE_PATH = os.path.join(HOME_CACHE_PATH, "custom")
-if not os.path.exists(CUSTOM_CACHE_PATH):
-    os.makedirs(CUSTOM_CACHE_PATH)
-    _init_file = os.path.join(CUSTOM_CACHE_PATH, "__init__.py")
+if not MODULES_CACHE_PATH:
+    MODULES_CACHE_PATH = os.path.join(HOME_CACHE_PATH, "modules")
+if not os.path.exists(MODULES_CACHE_PATH):
+    os.makedirs(MODULES_CACHE_PATH)
+    _init_file = os.path.join(MODULES_CACHE_PATH, "__init__.py")
     with open(_init_file, "w") as f:
         f.write("#")
-sys.path.append(os.path.dirname(CUSTOM_CACHE_PATH))
-MODEL_SAVE_NAME = "graph_model_{graph_id}"
+sys.path.append(os.path.dirname(MODULES_CACHE_PATH))
+MODEL_SAVE_NAME = "graph_model_{graph_id}_{object_id}"
 ALGO_MODEL_SAVE_NAME = "algo_model_{graph_id}_{node_id}_{algo_mod_type}_{idx}"
 
 # 运行模式 1: 包引用 2：grpc

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/constants.py

@@ -236,3 +236,186 @@ class ONLINE_ENV(ENUMSBASE):
 
 NODE_DATA_RESERVE = 20
 NOT_CUT_MODULES = []
+
+
+GRAPH_DICT_FORMAT = {
+    "id": {
+        "required": True,
+        "type": str,
+        "dest": "",
+        "help": "必传,不能包含中文",
+    },
+    "name": {
+        "required": False,
+        "type": str,
+        "dest": "",
+        "help": "非必传,不传参时,使用id的值",
+        "fillnone": "id",
+    },
+    "version": {
+        "required": False,
+        "default": "v0.0.1",
+        "type": str,
+        "dest": "",
+        "help": "非必传, 版本号",
+    },
+    "author": {
+        "required": False,
+        "default": "",
+        "type": str,
+        "dest": "username",
+        "help": "非必传,作者",
+    },
+    "update_time": {
+        "required": False,
+        "type": str,
+        "dest": "utime",
+        "help": "非必传,更新时间",
+    },
+    "params": {
+        "required": False,
+        "default": {},
+        "type": dict,
+        "dest": "run_params",
+        "help": "非必传，模型全局传参",
+    },
+    "nodes":{
+        "required": True,
+        "type": list,
+        "dest": "",
+        "help": "模型节点列表",
+        "elements": {
+            "required": True,
+            "type": dict,
+            "elements": {
+                "id": {
+                    "required": True,
+                    "type": str,
+                    "dest": "",
+                    "help": "必传,不能包含中文",
+                },
+                "name": {
+                    "required": False,
+                    "type": str,
+                    "dest": "",
+                    "help": "非必传,不传参时,使用id的值",
+                    "fillnone": "id",
+                },
+                "module_classification": {
+                    "required": True,
+                    "type": str,
+                    # "dest": "parent_cls_name",
+                    "help": "必传,多级时用‘/’分割",
+                },
+                "module_cls_name": {
+                    "required": True,
+                    "type": str,
+                    # "dest": "cls_name",
+                    "help": "必传,节点入口类的类名, 文件名为类名的小写",
+                },
+                "params": {
+                    "required": False,
+                    "default": {},
+                    "type": dict,
+                    "help": "必传,节点传参",
+                }
+            }
+        }
+    },
+    "edges":{
+        "required": False,
+        "default": [],
+        "type": list,
+        "dest": "",
+        "help": "模型连线列表",
+        "elements": {
+            "required": True,
+            "type": dict,
+            "elements": {
+                "source": {
+                    "required": True,
+                    "type": str,
+                    "dest": "source_id",
+                    "alias": "source_id",
+                    "help": "必传,起始节点id, 必须在nodes能找到对应id",
+                },
+                "target": {
+                    "required": True,
+                    "type": str,
+                    "dest": "target_id",
+                    "alias": "target_id",
+                    "help": "必传,目标节点id, 必须在nodes能找到对应id",
+                },
+                "source_anchor": {
+                    "required": False,
+                    "default": 1,
+                    "type": str,
+                    "dest": "",
+                    "help": "必传,起始节点锚点,默认是1",
+                },
+                "target_anchor": {
+                    "required": False,
+                    "default": 1,
+                    "type": str,
+                    "dest": "",
+                    "help": "必传,目标节点锚点,默认是1",
+                }
+            }
+        }
+    },
+    "objects":{
+        "required": True,
+        "dest": "object_infos",
+        "type": list,
+        "help": "模型对象信息",
+        "elements": {
+            "required": True,
+            "type": dict,
+            "elements": {
+                "id": {
+                    "required": True,
+                    "type": str,
+                    "dest": "",
+                    "help": "必传,不能包含中文",
+                },
+                "name": {
+                    "required": False,
+                    "type": str,
+                    "dest": "",
+                    "help": "非必传,不传参时,使用id的值",
+                    "fillnone": "id",
+                }
+            }
+        }
+    }
+}
+
+
+EXECUTOR_DICT_FORMAT = {
+    "executor":{
+        "required": False,  # 没有执行器的时候模型安顺序执行
+        "default": [],
+        "type": list,
+        "help": "执行器,标识模型执行顺序",
+        "elements": {
+            "required": True,
+            "type": dict,
+            "elements": {
+                "source": {
+                    "required": True,
+                    "type": str,
+                    "dest": "source_graph_id",
+                    "alias": "source_id",
+                    "help": "必传,起始模型id",
+                },
+                "target": {
+                    "required": True,
+                    "type": str,
+                    "dest": "target_graph_id",
+                    "alias": "target_id",
+                    "help": "必传,目标模型id",
+                }
+            }
+        }
+    }
+}

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/__init__.py

@@ -4,7 +4,6 @@
 from datetime import datetime, timedelta
 from PipeGraphPy.utils.str_handle import filter_fields
 from PipeGraphPy.utils.handle_graph import stop_running_with_exception
-from PipeGraphPy.logger import plog, rlog, log
 
 
 class Base(object):
@@ -21,23 +20,6 @@ class MBase(object):
         """停止运行"""
         stop_running_with_exception(self.params.graph_info["id"], err=err)
 
-    def log(self, l_type=None, level="info", msg=None):
-        """打印日志"""
-        if l_type == "r":
-            glog = rlog
-        elif l_type == "p":
-            glog = plog
-        else:
-            glog = log
-        func = getattr(glog, level)
-        func(
-            "%s %s: %s" % ((datetime.utcnow()+timedelta(hours=8)), self.__class__.__name__, msg),
-            graph_id=self.params.graph_info["id"],
-            plog_record_id=self.params.get("plog_record_id", 0),
-            source=str(self.params.get("source")),
-            clock=str(self.params.get("clock")),
-        )
-
     def get_X_y(self, df):
         """获取X和y值从df中"""
         if self.params.get("feature_columns"):

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/branchselect/sqlbranchselect.py

@@ -8,7 +8,7 @@
 """
 
 import pandas as pd
-from custom.preprocessor.sqlfilter import SqlFilter
+from modules.preprocessor.sqlfilter import SqlFilter
 
 
 class SqlBranchSelect():

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/evaluate/rmseeval.py

@@ -40,7 +40,7 @@ class TotalEvaluateData():
                 nwp_config[k] = list(set(nwp_config[k]) | set(v))
             feature = list(set(feature) | set(n.params["feature"]))
             check_result = list(set(check_result) | set(n.params.get("check_result")))
-        graph = Graph.create(wfid=self.evaluate_graph_info['objects_id'])
+        graph = Graph.create(wfid=self.evaluate_graph_info['object_id'])
         algo_data_node = Node.create(Module.use("Algodata"), graph)
         algo_data_node.params = {
                 "out_col": out_col,
@@ -242,7 +242,7 @@ class RmseEval():
     }
     def __init__(self, **kw):
         self.params = kw
-        self.farm_info = self.params.get('objects_info', {})
+        self.farm_info = self.params.get('object_info', {})
         self.graph_info = self.params.get('graph_info', {})
         self.powercap = self.farm_info.get('powercap', POWERCAP)
         self.theory_score = 50

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/importdata/__init__.py

@@ -7,7 +7,7 @@ import re
 from PipeGraphPy.constants import FARMTYPE
 # from algo_data.utils import inp_heights
 # import algo_data
-from PipeGraphPy.logger import plog
+from PipeGraphPy.logger import log
 from collections import namedtuple
 from PipeGraphPy.config import settings
 from datetime import datetime, timedelta
@@ -124,8 +124,8 @@ class AlgodataBase(InputDataBase):
         InputDataBase.__init__(self, **kw)
         self.params = kw
         self.graph_info = kw.get("graph_info")
-        self.wfid = self.graph_info["objects_id"]
-        self.farm_info = kw.get("objects_info")
+        self.wfid = self.graph_info["object_id"]
+        self.farm_info = kw.get("object_info")
         default_feature = (
             "ghi_sfc"
             if self.farm_info["f_type"] == FARMTYPE.PV
@@ -399,7 +399,7 @@ class AlgodataBase(InputDataBase):
 #             weather_data = weather_data.sort_index(axis=1)
 #             return weather_data
 #         except Exception as e:
-#             plog.info(e)
+#             log.info(e)
 #             if is_except:
 #                 raise e
 #             else:
@@ -642,7 +642,7 @@ class AlgodataBase(InputDataBase):
 #             weather_df = weather_df.sort_index(axis=1)
 #             return weather_df
 #         except Exception as e:
-#             plog.info(e)
+#             log.info(e)
 #             if is_except:
 #                 raise e
 #             else:

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/importdata/ensemblegridweather.py

@@ -190,8 +190,8 @@ class EnsembleGridWeather(InputDataBase):
     def __init__(self, **kw):
         InputDataBase.__init__(self, **kw)
         graph_info = kw["graph_info"]
-        self.wfid = graph_info["objects_id"]
-        self.farm_info = self.params["objects_info"]
+        self.wfid = graph_info["object_id"]
+        self.farm_info = self.params["object_info"]
 
     def _get_obs_data(self, start_dt, end_dt, columns=["r_apower"]):
         """调用algo_data接口，获取数据库obs_data实测数据"""

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/importdata/fromsql.py

@@ -25,7 +25,7 @@ class FromSQL(InputDataBase):
         "type": "string",
         "plugin": "text",
         "need": True,
-        "value": "select * from PipeGraphPy.custom where mod_id=20",
+        "value": "select * from PipeGraphPy.modules where mod_id=20",
         "desc": ""
     }, {
         "key": "index",

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/importdata/gridweather.py

@@ -207,8 +207,8 @@ class GridWeather(InputDataBase):
     def __init__(self, **kw):
         InputDataBase.__init__(self, **kw)
         graph_info = kw["graph_info"]
-        self.wfid = graph_info["objects_id"]
-        self.farm_info = self.params["objects_info"]
+        self.wfid = graph_info["object_id"]
+        self.farm_info = self.params["object_info"]
 
     def _get_obs_data(self, start_dt, end_dt, columns=["r_apower"]):
         """调用algo_data接口，获取数据库obs_data实测数据"""

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/importdata/qiaogdfsimport.py

@@ -100,16 +100,16 @@ class QiaoGDFSImport():
     def __init__(self,**kw):
         # super().__init__(**kw)
         self.params = kw
-        self.farm_info = kw.get("objects_info", {})
+        self.farm_info = kw.get("object_info", {})
         self.datapath = "/mnt/d/use_data" # 本地测试使用
         # self.datapath = "/jtdata/products/data" # 线上的数据路径
-        self.gdfs_path_history="/JTC3WS8V/GDFS/merge/%s/" % (self.datapath, self.farm_info.get("objects_id"))
-        self.ec_path_history="%s/JTE72HN5/EC/merge/%s/" % (self.datapath, self.farm_info.get("objects_id"))
+        self.gdfs_path_history="/JTC3WS8V/GDFS/merge/%s/" % (self.datapath, self.farm_info.get("object_id"))
+        self.ec_path_history="%s/JTE72HN5/EC/merge/%s/" % (self.datapath, self.farm_info.get("object_id"))
         self.gdfs_path_Realtime="%s/JT6DE34S/" % self.datapath
         self.ec_path_Realtime = "%s/JT1A98G9/" % self.datapath
-        self.ws_path=r"%s/JTLMOXVE/%s/ws.csv" % (self.datapath, self.farm_info.get("objects_id"))
-        self.sr_path=r"%s/JTLMOXVE/%s/sr.csv" % (self.datapath, self.farm_info.get("objects_id"))
-        self.power_path = r"%s/JTLMOXVE/%s/power.csv" % (self.datapath, self.farm_info.get("objects_id"))
+        self.ws_path=r"%s/JTLMOXVE/%s/ws.csv" % (self.datapath, self.farm_info.get("object_id"))
+        self.sr_path=r"%s/JTLMOXVE/%s/sr.csv" % (self.datapath, self.farm_info.get("object_id"))
+        self.power_path = r"%s/JTLMOXVE/%s/power.csv" % (self.datapath, self.farm_info.get("object_id"))
         self.farm_type = self.farm_info.get("ext_type_1")
 
     def cal_data_range(self,  days, reserve=0, last_dt=None):
@@ -416,7 +416,7 @@ class QiaoGDFSImport():
         """
         预测gdfs数据处理
         """
-        file_tail_str = "%s.rb" % self.farm_info.get("objects_id")
+        file_tail_str = "%s.rb" % self.farm_info.get("object_id")
         data =pd.DataFrame([])
         now_time= (datetime.utcnow()+timedelta(hours=8)).strftime("%Y%m%d")
         if not os.path.exists(os.path.join(self.gdfs_path_Realtime,now_time)):
@@ -440,7 +440,7 @@ class QiaoGDFSImport():
         """
         预测EC数据数据
         """
-        file_tail_str = "%s.rb" % self.farm_info.get("objects_id")
+        file_tail_str = "%s.rb" % self.farm_info.get("object_id")
         data =pd.DataFrame([])
         now_time= ((datetime.utcnow()+timedelta(hours=8)) - timedelta(days=1) ).strftime("%Y%m%d")  # 获取上一天20批次数据
         if not os.path.exists(os.path.join(self.ec_path_Realtime,now_time)):

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/importdata/shan1xiareaimport.py

@@ -99,7 +99,7 @@ class Shan1XiAreaImport():
     def __init__(self, **kw):
         # super().__init__(**kw)
         self.params = kw
-        self.farm_info = kw.get("objects_info", {})
+        self.farm_info = kw.get("object_info", {})
         self.data_path = ""
         self.datapath = "/mnt/d/use_data" # 本地测试使用
         # self.datapath = "/jtdata/products/data" # 线上的数据路径
@@ -371,7 +371,7 @@ class Shan1XiAreaImport():
         """
         预测cfs45数据数据
         """
-        file_tail_str = "%s.rb" % self.farm_info.get("objects_id")  # 风场或者区域farm_id
+        file_tail_str = "%s.rb" % self.farm_info.get("object_id")  # 风场或者区域farm_id
         data_con = pd.DataFrame([])
         now_time = datetime.utcnow()
         now_time_strf = now_time.strftime("%Y%m%d") + "0000"

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/importdata/strategyalgodata.py

@@ -12,7 +12,7 @@ from . import AlgodataBase
 from dateutil.relativedelta import relativedelta
 from dbpoolpy import connect_db, Select
 from collections import defaultdict
-from PipeGraphPy.logger import plog
+from PipeGraphPy.logger import log
 from PipeGraphPy.config import settings
 from PipeGraphPy.core.module import Module
 from PipeGraphPy.core.node import Node
@@ -447,7 +447,7 @@ class StrategyAlgodata(AlgodataBase):
     #         weather_data = weather_data.sort_index(axis=1)
     #         return weather_data
     #     except Exception as e:
-    #         plog.info(e)
+    #         log.info(e)
     #         if is_except:
     #             raise e
     #         else:

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/importdata/theorydata.py

@@ -8,7 +8,7 @@ from PipeGraphPy.constants import FARMTYPE
 from PipeGraphPy.constants import DATATYPE
 from PipeGraphPy.config import settings
 from . import AlgodataBase
-from PipeGraphPy.logger import plog
+from PipeGraphPy.logger import log
 
 nwp_order = ["EC", "CMA", "GFS", "MIX", "METE", "SUP", "OPT"]
 
@@ -99,7 +99,7 @@ class TheoryData(AlgodataBase):
                         continue
                     break
                 except Exception as e:
-                    plog.info(str(e))
+                    log.info(str(e))
         if nwp_data.empty:
             raise Exception("所有气象源均无数据%s失败" % self.params["nwp_config"])
         if self.farm_info["f_type"] == FARMTYPE.WIND:

{pipegraphpy-2.0.3 → pipegraphpy-2.0.5}/src/PipeGraphPy/core/modules/importdata/totalpower.py

@@ -87,7 +87,7 @@ class TotalPower(InputDataBase):
         InputDataBase.__init__(self, **kw)
         self.params = kw
         self.graph_info = kw.get("graph_info")
-        self.farm_info = kw.get("objects_info")
+        self.farm_info = kw.get("object_info")
 
     def run(self):
         """运行,取训练数据"""
@@ -115,7 +115,7 @@ class TotalPower(InputDataBase):
             f_type=self.farm_info['f_type'],
             dtype='df',
         )
-        info_w = info_w[info_w["wfid"]!=self.params["objects_info"]['wfid']]
+        info_w = info_w[info_w["wfid"]!=self.params["object_info"]['wfid']]
         wfids = info_w['wfid'].values.tolist()
         if not wfids:
             raise Exception("未找到电场")