PipeGraphPy 2.0.4__tar.gz → 2.0.5b1__tar.gz

pipegraphpy-2.0.5b1/.gitignore

@@ -0,0 +1,26 @@
+*.pyc
+#*.txt
+# *.csv
+imlf/json_files/*
+checkpoint
+mymodel.tflearn.*
+*.egg-info
+dist
+*.log
+.vscode/*
+.cache
+build/*
+json_files/*
+algo_data/*
+.pytest_cache/
+.scannerwork/
+sonar-report/
+.coverage
+__pycache__/
+structured_data_regressor/
+model_files/*
+logs/*
+*.pid
+demo/*
+tests/*
+competition/*

pipegraphpy-2.0.5b1/.pypi_token

@@ -0,0 +1,3 @@
+UploadAllProj
+pypi-AgEIcHlwaS5vcmcCJGE0MDhjYWE4LWRmNmQtNGYzMC1iMGI5LThlMTY4OWRjNDRhZQACKlszLCJlNTlhODZmMi1hMmUwLTRmM2QtOTEzMS0wNzUzMmUyOTZiY2QiXQAABiD3Wap-15uU01YIt5_XMMn2v_8bwojhghDi1Q446SDqwg
+

{pipegraphpy-2.0.4/src/PipeGraphPy.egg-info → pipegraphpy-2.0.5b1}/PKG-INFO

@@ -1,22 +1,26 @@
-Metadata-Version: 2.1
-Name: PipeGraphPy
-Version: 2.0.4
-Summary: 核心算法框架
-Home-page: https://gitee.com/zsq_tee/pipe-graph-py
-Author: liujm
-Author-email: 792079829@qq.com
-License: Private
-Project-URL: Documentation, https://gitee.com/zsq_tee/pipe-graph-py
-Project-URL: Source, https://gitee.com/zsq_tee/pipe-graph-py
-Requires-Python: >=3.6
-Requires-Dist: openpyxl
-Requires-Dist: pandas
-Requires-Dist: python-dateutil
-Requires-Dist: PyYAML
-Requires-Dist: tqdm
-Requires-Dist: numpy
-Requires-Dist: matplotlib
-Requires-Dist: pandasql
-Requires-Dist: dnspython
-Requires-Dist: prettytable
-Requires-Dist: dbpoolpy
+Metadata-Version: 2.1
+Name: PipeGraphPy
+Version: 2.0.5b1
+Summary: 核心算法框架
+Home-page: https://gitee.com/zsq_tee/pipe-graph-py
+Author: liujm
+Author-email: 792079829@qq.com
+License: Private
+Project-URL: Documentation, https://gitee.com/zsq_tee/pipe-graph-py
+Project-URL: Source, https://gitee.com/zsq_tee/pipe-graph-py
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: MacOS
+Requires-Python: >=3.6
+Requires-Dist: openpyxl
+Requires-Dist: pandas
+Requires-Dist: python-dateutil
+Requires-Dist: PyYAML
+Requires-Dist: tqdm
+Requires-Dist: numpy
+Requires-Dist: matplotlib
+Requires-Dist: pandasql
+Requires-Dist: dnspython
+Requires-Dist: prettytable
+Requires-Dist: dbpoolpy

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/README.md

@@ -87,7 +87,7 @@ class ImportExample():
 
     def __init__(self, **kw):
         self.params = kw
-        self.farm_info = kw.get("objects_info", {})
+        self.farm_info = kw.get("object_info", {})
 
     def run(self):
         df = pd.DataFrame(
@@ -371,7 +371,7 @@ class SqlFilter():
     __version__ = '0.0.2'
     def __init__(self, **kw):
         self.params = kw
-        self.farm_info = kw.get('objects_info')
+        self.farm_info = kw.get('object_info')
 
     def _sqlfilter(self, sql, X, y=None):
         X, y = X, pd.DataFrame() if y is None else y

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/setup.cfg

@@ -1,40 +1,40 @@
-[metadata]
-name = PipeGraphPy
-version = attr: PipeGraphPy.__version__
-author = liujm
-author_email = 792079829@qq.com
-url = https://gitee.com/zsq_tee/pipe-graph-py
-description = A Python library of core algorithms for weather group forecasting
-long_description = file: README.rst
-license = MIT
-project_urls = 
-	Documentation = https://gitee.com/zsq_tee/pipe-graph-py
-	Source = https://gitee.com/zsq_tee/pipe-graph-py
-
-[options]
-packages = find:
-package_dir = = src
-include_package_data = true
-python_requires = >= 3.6
-
-[options.packages.find]
-where = src
-
-[options.entry_points]
-
-[flake8]
-select = B, E, F, W, B9, ISC
-ignore = 
-	E203
-	E402
-	E501
-	E722
-	W503
-max-line-length = 80
-per-file-ignores = 
-	src/flask/__init__.py: F401
-
-[egg_info]
-tag_build = 
-tag_date = 0
-
+[metadata]
+name = PipeGraphPy
+version = attr: PipeGraphPy.__version__
+author = liujm
+author_email = 792079829@qq.com
+url = https://gitee.com/zsq_tee/pipe-graph-py
+description = A Python library of core algorithms for weather group forecasting
+long_description = file: README.rst
+license = MIT
+project_urls = 
+	Documentation = https://gitee.com/zsq_tee/pipe-graph-py
+	Source = https://gitee.com/zsq_tee/pipe-graph-py
+
+[options]
+packages = find:
+package_dir = = src
+include_package_data = true
+python_requires = >= 3.6
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+
+[flake8]
+select = B, E, F, W, B9, ISC
+ignore = 
+	E203
+	E402
+	E501
+	E722
+	W503
+max-line-length = 80
+per-file-ignores = 
+	src/flask/__init__.py: F401
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/setup.py

@@ -19,4 +19,10 @@ setup(
     license="Private",
     include_package_data=True,
     install_requires=get_requirements(),
+    classifiers=[
+        'Programming Language :: Python :: 3.6',
+        'Operating System :: POSIX :: Linux',
+        'Operating System :: Microsoft :: Windows',
+        'Operating System :: MacOS',
+    ],
 )

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/__init__.py

@@ -1,6 +1,6 @@
 # coding:utf-8
 
-__version__ = "2.0.4"
+__version__ = "2.0.5b1"
 
 from PipeGraphPy.core.graph import Graph, graph_predict, graph_evaluate, online_graph_evaluate, online_graph_predict, graph_run, graph_backtest
 from PipeGraphPy.core.module import Module

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/config/default_settings.py

@@ -30,7 +30,7 @@ BACKTEST_RESULT_SAVE_PATH = os.environ.get("PGP_BACKTEST_RESULT_SAVE_PATH", '')
 RUN_RESULT_SAVE_PATH = os.environ.get("PGP_RUN_RESULT_SAVE_PATH", '')
 EVALUATE_RESULT_SAVE_PATH = os.environ.get("PGP_EVALUATE_RESULT_SAVE_PATH", '')
 TEMP_SAVE_PATH = os.environ.get("PGP_TEMP_SAVE_PATH", "")
-CUSTOM_CACHE_PATH = os.environ.get("PGP_CUSTOM_CACHE_PATH", '')
+MODULES_CACHE_PATH = os.environ.get("PGP_MODULES_CACHE_PATH", '')
 REDIS_HOST = os.environ.get("REDIS_HOST", '39.105.185.60')
 REDIS_PORT = int(os.environ.get("REDIS_PORT", '6371'))
 PGP_DB_HOST = os.environ.get("PGP_DB_HOST", '39.105.185.60')
@@ -68,15 +68,15 @@ if not EVALUATE_RESULT_SAVE_PATH:
     EVALUATE_RESULT_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "evaluate_result")
 if not TEMP_SAVE_PATH:
     TEMP_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "temp")
-if not CUSTOM_CACHE_PATH:
-    CUSTOM_CACHE_PATH = os.path.join(HOME_CACHE_PATH, "custom")
-if not os.path.exists(CUSTOM_CACHE_PATH):
-    os.makedirs(CUSTOM_CACHE_PATH)
-    _init_file = os.path.join(CUSTOM_CACHE_PATH, "__init__.py")
+if not MODULES_CACHE_PATH:
+    MODULES_CACHE_PATH = os.path.join(HOME_CACHE_PATH, "modules")
+if not os.path.exists(MODULES_CACHE_PATH):
+    os.makedirs(MODULES_CACHE_PATH)
+    _init_file = os.path.join(MODULES_CACHE_PATH, "__init__.py")
     with open(_init_file, "w") as f:
         f.write("#")
-sys.path.append(os.path.dirname(CUSTOM_CACHE_PATH))
-MODEL_SAVE_NAME = "graph_model_{graph_id}"
+sys.path.append(os.path.dirname(MODULES_CACHE_PATH))
+MODEL_SAVE_NAME = "graph_model_{graph_id}_{object_id}"
 ALGO_MODEL_SAVE_NAME = "algo_model_{graph_id}_{node_id}_{algo_mod_type}_{idx}"
 
 # 运行模式 1: 包引用 2：grpc

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/constants.py

@@ -236,3 +236,186 @@ class ONLINE_ENV(ENUMSBASE):
 
 NODE_DATA_RESERVE = 20
 NOT_CUT_MODULES = []
+
+
+GRAPH_DICT_FORMAT = {
+    "id": {
+        "required": True,
+        "type": str,
+        "dest": "",
+        "help": "必传,不能包含中文",
+    },
+    "name": {
+        "required": False,
+        "type": str,
+        "dest": "",
+        "help": "非必传,不传参时,使用id的值",
+        "fillnone": "id",
+    },
+    "version": {
+        "required": False,
+        "default": "v0.0.1",
+        "type": str,
+        "dest": "",
+        "help": "非必传, 版本号",
+    },
+    "author": {
+        "required": False,
+        "default": "",
+        "type": str,
+        "dest": "username",
+        "help": "非必传,作者",
+    },
+    "update_time": {
+        "required": False,
+        "type": str,
+        "dest": "utime",
+        "help": "非必传,更新时间",
+    },
+    "params": {
+        "required": False,
+        "default": {},
+        "type": dict,
+        "dest": "run_params",
+        "help": "非必传，模型全局传参",
+    },
+    "nodes":{
+        "required": True,
+        "type": list,
+        "dest": "",
+        "help": "模型节点列表",
+        "elements": {
+            "required": True,
+            "type": dict,
+            "elements": {
+                "id": {
+                    "required": True,
+                    "type": str,
+                    "dest": "",
+                    "help": "必传,不能包含中文",
+                },
+                "name": {
+                    "required": False,
+                    "type": str,
+                    "dest": "",
+                    "help": "非必传,不传参时,使用id的值",
+                    "fillnone": "id",
+                },
+                "module_classification": {
+                    "required": True,
+                    "type": str,
+                    # "dest": "parent_cls_name",
+                    "help": "必传,多级时用‘/’分割",
+                },
+                "module_cls_name": {
+                    "required": True,
+                    "type": str,
+                    # "dest": "cls_name",
+                    "help": "必传,节点入口类的类名, 文件名为类名的小写",
+                },
+                "params": {
+                    "required": False,
+                    "default": {},
+                    "type": dict,
+                    "help": "必传,节点传参",
+                }
+            }
+        }
+    },
+    "edges":{
+        "required": False,
+        "default": [],
+        "type": list,
+        "dest": "",
+        "help": "模型连线列表",
+        "elements": {
+            "required": True,
+            "type": dict,
+            "elements": {
+                "source": {
+                    "required": True,
+                    "type": str,
+                    "dest": "source_id",
+                    "alias": "source_id",
+                    "help": "必传,起始节点id, 必须在nodes能找到对应id",
+                },
+                "target": {
+                    "required": True,
+                    "type": str,
+                    "dest": "target_id",
+                    "alias": "target_id",
+                    "help": "必传,目标节点id, 必须在nodes能找到对应id",
+                },
+                "source_anchor": {
+                    "required": False,
+                    "default": 1,
+                    "type": str,
+                    "dest": "",
+                    "help": "必传,起始节点锚点,默认是1",
+                },
+                "target_anchor": {
+                    "required": False,
+                    "default": 1,
+                    "type": str,
+                    "dest": "",
+                    "help": "必传,目标节点锚点,默认是1",
+                }
+            }
+        }
+    },
+    "objects":{
+        "required": True,
+        "dest": "object_infos",
+        "type": list,
+        "help": "模型对象信息",
+        "elements": {
+            "required": True,
+            "type": dict,
+            "elements": {
+                "id": {
+                    "required": True,
+                    "type": str,
+                    "dest": "",
+                    "help": "必传,不能包含中文",
+                },
+                "name": {
+                    "required": False,
+                    "type": str,
+                    "dest": "",
+                    "help": "非必传,不传参时,使用id的值",
+                    "fillnone": "id",
+                }
+            }
+        }
+    }
+}
+
+
+EXECUTOR_DICT_FORMAT = {
+    "executor":{
+        "required": False,  # 没有执行器的时候模型安顺序执行
+        "default": [],
+        "type": list,
+        "help": "执行器,标识模型执行顺序",
+        "elements": {
+            "required": True,
+            "type": dict,
+            "elements": {
+                "source": {
+                    "required": True,
+                    "type": str,
+                    "dest": "source_graph_id",
+                    "alias": "source_id",
+                    "help": "必传,起始模型id",
+                },
+                "target": {
+                    "required": True,
+                    "type": str,
+                    "dest": "target_graph_id",
+                    "alias": "target_id",
+                    "help": "必传,目标模型id",
+                }
+            }
+        }
+    }
+}

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/core/modules/__init__.py

@@ -4,7 +4,6 @@
 from datetime import datetime, timedelta
 from PipeGraphPy.utils.str_handle import filter_fields
 from PipeGraphPy.utils.handle_graph import stop_running_with_exception
-from PipeGraphPy.logger import plog, rlog, log
 
 
 class Base(object):
@@ -21,23 +20,6 @@ class MBase(object):
         """停止运行"""
         stop_running_with_exception(self.params.graph_info["id"], err=err)
 
-    def log(self, l_type=None, level="info", msg=None):
-        """打印日志"""
-        if l_type == "r":
-            glog = rlog
-        elif l_type == "p":
-            glog = plog
-        else:
-            glog = log
-        func = getattr(glog, level)
-        func(
-            "%s %s: %s" % ((datetime.utcnow()+timedelta(hours=8)), self.__class__.__name__, msg),
-            graph_id=self.params.graph_info["id"],
-            plog_record_id=self.params.get("plog_record_id", 0),
-            source=str(self.params.get("source")),
-            clock=str(self.params.get("clock")),
-        )
-
     def get_X_y(self, df):
         """获取X和y值从df中"""
         if self.params.get("feature_columns"):

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/core/modules/branchselect/sqlbranchselect.py

@@ -8,7 +8,7 @@
 """
 
 import pandas as pd
-from custom.preprocessor.sqlfilter import SqlFilter
+from modules.preprocessor.sqlfilter import SqlFilter
 
 
 class SqlBranchSelect():

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/core/modules/evaluate/rmseeval.py

@@ -40,7 +40,7 @@ class TotalEvaluateData():
                 nwp_config[k] = list(set(nwp_config[k]) | set(v))
             feature = list(set(feature) | set(n.params["feature"]))
             check_result = list(set(check_result) | set(n.params.get("check_result")))
-        graph = Graph.create(wfid=self.evaluate_graph_info['objects_id'])
+        graph = Graph.create(wfid=self.evaluate_graph_info['object_id'])
         algo_data_node = Node.create(Module.use("Algodata"), graph)
         algo_data_node.params = {
                 "out_col": out_col,
@@ -242,7 +242,7 @@ class RmseEval():
     }
     def __init__(self, **kw):
         self.params = kw
-        self.farm_info = self.params.get('objects_info', {})
+        self.farm_info = self.params.get('object_info', {})
         self.graph_info = self.params.get('graph_info', {})
         self.powercap = self.farm_info.get('powercap', POWERCAP)
         self.theory_score = 50

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/core/modules/importdata/__init__.py

@@ -7,7 +7,7 @@ import re
 from PipeGraphPy.constants import FARMTYPE
 # from algo_data.utils import inp_heights
 # import algo_data
-from PipeGraphPy.logger import plog
+from PipeGraphPy.logger import log
 from collections import namedtuple
 from PipeGraphPy.config import settings
 from datetime import datetime, timedelta
@@ -124,8 +124,8 @@ class AlgodataBase(InputDataBase):
         InputDataBase.__init__(self, **kw)
         self.params = kw
         self.graph_info = kw.get("graph_info")
-        self.wfid = self.graph_info["objects_id"]
-        self.farm_info = kw.get("objects_info")
+        self.wfid = self.graph_info["object_id"]
+        self.farm_info = kw.get("object_info")
         default_feature = (
             "ghi_sfc"
             if self.farm_info["f_type"] == FARMTYPE.PV
@@ -399,7 +399,7 @@ class AlgodataBase(InputDataBase):
 #             weather_data = weather_data.sort_index(axis=1)
 #             return weather_data
 #         except Exception as e:
-#             plog.info(e)
+#             log.info(e)
 #             if is_except:
 #                 raise e
 #             else:
@@ -642,7 +642,7 @@ class AlgodataBase(InputDataBase):
 #             weather_df = weather_df.sort_index(axis=1)
 #             return weather_df
 #         except Exception as e:
-#             plog.info(e)
+#             log.info(e)
 #             if is_except:
 #                 raise e
 #             else:

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/core/modules/importdata/ensemblegridweather.py

@@ -190,8 +190,8 @@ class EnsembleGridWeather(InputDataBase):
     def __init__(self, **kw):
         InputDataBase.__init__(self, **kw)
         graph_info = kw["graph_info"]
-        self.wfid = graph_info["objects_id"]
-        self.farm_info = self.params["objects_info"]
+        self.wfid = graph_info["object_id"]
+        self.farm_info = self.params["object_info"]
 
     def _get_obs_data(self, start_dt, end_dt, columns=["r_apower"]):
         """调用algo_data接口，获取数据库obs_data实测数据"""

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/core/modules/importdata/fromsql.py

@@ -25,7 +25,7 @@ class FromSQL(InputDataBase):
         "type": "string",
         "plugin": "text",
         "need": True,
-        "value": "select * from PipeGraphPy.custom where mod_id=20",
+        "value": "select * from PipeGraphPy.modules where mod_id=20",
         "desc": ""
     }, {
         "key": "index",

{pipegraphpy-2.0.4 → pipegraphpy-2.0.5b1}/src/PipeGraphPy/core/modules/importdata/gridweather.py

@@ -207,8 +207,8 @@ class GridWeather(InputDataBase):
     def __init__(self, **kw):
         InputDataBase.__init__(self, **kw)
         graph_info = kw["graph_info"]
-        self.wfid = graph_info["objects_id"]
-        self.farm_info = self.params["objects_info"]
+        self.wfid = graph_info["object_id"]
+        self.farm_info = self.params["object_info"]
 
     def _get_obs_data(self, start_dt, end_dt, columns=["r_apower"]):
         """调用algo_data接口，获取数据库obs_data实测数据"""