PipeGraphPy 2.0.6__py3-none-win_amd64.whl

PipeGraphPy/utils/core.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import datetime
+import pandas as pd
+from collections import defaultdict
+from PipeGraphPy.core.node import Node
+from PipeGraphPy.db.models import NodesTB, EdgesTB
+from PipeGraphPy.core.module import get_template_type
+
+# from prettytable import PrettyTable
+
+
+def get_start_and_end(nodes_group_dict):
+    """找到所有开始节点和结束节点
+    parameters:
+        nodes_group_dict: dict 节点关系列表(直接从数据库取出的列表)
+    return
+        开始节点集和结束节点集
+    """
+    from_set, to_set = set(), set()
+    for source_id, target_id in nodes_group_dict.keys():
+        from_set.add(source_id)
+        to_set.add(target_id)
+    return from_set - to_set, to_set - from_set
+
+
+def group_by_source_node(nodes_group_dict):
+    """统计每个节点下子节点
+    parameters:
+        edges_dict: dict 节点关系列表
+
+    """
+    # 统计每个节点下的所有子节点
+    source_nodes_group = defaultdict(list)
+    for source_id, target_id in nodes_group_dict.keys():
+        source_nodes_group[source_id].append(target_id)
+    return source_nodes_group
+
+
+def is_cycle2(edges, nodes):
+    """判断是否有环：单个节点在一个单向线路中只能出现一回
+    parameters:
+        edges: list 边列表
+        nodes: list 节点列表
+    return
+        Boolen
+    """
+    nodes_dict = {i["id"]: i for i in nodes}
+    nodes_group_dict = gen_nodes_group_dict2(edges, nodes_dict)
+
+    # 找出开始节点
+    start_set, end_set = get_start_and_end(nodes_group_dict)
+    if not start_set or not end_set:
+        return True
+    # 统计每个节点下的所有子节点
+    source_nodes_group = group_by_source_node(nodes_group_dict)
+    # 循环查看单向线路中节点是否重复出现
+    res = False
+    node_set = set()
+
+    def _loop(node):
+        nonlocal res, node_set, source_nodes_group
+        if node in node_set:
+            res = True
+            return
+        node_set.add(node)
+        for i in source_nodes_group[node]:
+            _loop(i)
+            if res:
+                return
+            node_set.remove(i)
+
+    # 以开始节点作业起点判断
+    for head in start_set:
+        if res:
+            break
+        _loop(head)
+
+    return res
+
+
+def is_cycle(graph_id, edges_info=None, nodes_info=None):
+    """判断是否有环：单个节点在一个单向线路中只能出现一回
+    parameters:
+        graph_id: int 图id
+        edges_info: list 边列表
+        nodes_info: list 节点列表
+    return
+        Boolen
+    """
+    if not edges_info:
+        edges_info = EdgesTB.find(graph_id=graph_id)
+    if not nodes_info:
+        nodes_info = NodesTB.find(graph_id=graph_id)
+    if not edges_info or not nodes_info:
+        return False
+    nodes_info_dict = {i["id"]: i for i in nodes_info}
+    nodes_group_dict = gen_nodes_group_dict(edges_info, nodes_info_dict)
+
+    # 找出开始节点
+    start_set, end_set = get_start_and_end(nodes_group_dict)
+    if not start_set or not end_set:
+        return True
+    # 统计每个节点下的所有子节点
+    source_nodes_group = group_by_source_node(nodes_group_dict)
+    # 循环查看单向线路中节点是否重复出现
+    res = False
+    node_set = set()
+
+    def _loop(node):
+        nonlocal res, node_set, source_nodes_group
+        if node in node_set:
+            res = True
+            return
+        node_set.add(node)
+        for i in source_nodes_group[node]:
+            _loop(i)
+            if res:
+                return
+            node_set.remove(i)
+
+    # 以开始节点作业起点判断
+    for head in start_set:
+        if res:
+            break
+        _loop(head)
+
+    return res
+
+
+def _add_node_loop(head, relation_dict, all_node, node_dict, graph_info):
+    """递归添加节点"""
+    if not relation_dict[head.id]:
+        return
+    for i in relation_dict[head.id]:
+        child_element = Node(i, info=node_dict[i], graph_info=graph_info)
+        # 如果已添加过，使用之前的对像
+        if all_node.get(child_element) is not None:
+            child_element = all_node[child_element]
+        else:
+            all_node[child_element] = child_element
+        if head not in child_element.fathers:
+            child_element.add_one_father(head)
+        if child_element not in head.children:
+            head.add_one_child(child_element)
+        _add_node_loop(child_element, relation_dict, all_node, node_dict, graph_info)
+
+
+def gen_node_heads(nodes_group_dict, nodes_info_dict, graph_info):
+    """生成节点并返回头节点
+    parameters:
+        edges_dict: dict 节点边列表
+    return:
+        头节点列表
+    """
+    # 找出开始节点
+    start_set, _ = get_start_and_end(nodes_group_dict)
+
+    # 统计每个节点下的所有子节点
+    source_nodes_group = group_by_source_node(nodes_group_dict)
+
+    head_list = [
+        Node(i, info=nodes_info_dict[i], graph_info=graph_info) for i in start_set
+    ]
+    head_list_copy = head_list
+    # 以头节点为开始节点循环添加子节点
+    all_node = dict()
+    for head in head_list:
+        _add_node_loop(head, source_nodes_group, all_node, nodes_info_dict, graph_info)
+    return head_list_copy
+
+
+def load_obj(source_str):
+    tmp_path = source_str.split(".")
+    op_str = tmp_path.pop()
+    import_str = ".".join(tmp_path)
+    try:
+        exec("from {} import {}".format(import_str, op_str))
+        op_obj = eval(op_str)
+    except Exception as e:
+        raise Exception(e)
+
+    return op_obj
+
+
+def outanchor_to_outidx(node_info, outanchor):
+    """输出的锚点转成输出的索引"""
+    input_data_type = get_template_type(node_info["mod_id"], "INPUT")
+    return outanchor - len(input_data_type)
+
+
+def outanchor_to_outidx2(node, outanchor):
+    """输出的锚点转成输出的索引"""
+    return node.outanchor_to_outidx(outanchor)
+
+
+def outidx_to_outanchor(node_info, outidx):
+    """输出的索引转成输出的锚点"""
+    input_data_type = get_template_type(node_info["mod_id"], "INPUT")
+    return outidx + len(input_data_type)
+
+
+def gen_nodes_group_dict(edges_info, nodes_info_dict):
+    """组合节点传参
+    return:
+        {(source_id, target_id): pass_idx}
+    """
+    out_idx = defaultdict(list)
+    in_idx = defaultdict(list)
+    for edge_info in edges_info:
+        # source_id找到对应结点类型
+        out_idx[(edge_info["source_id"], edge_info["target_id"])].append(
+            str(
+                outanchor_to_outidx(
+                    nodes_info_dict[edge_info["source_id"]], edge_info["source_anchor"]
+                )
+            )
+        )
+        in_idx[(edge_info["source_id"], edge_info["target_id"])].append(
+            str(edge_info["target_anchor"])
+        )
+    nodes_group_dict = {
+        k: ",".join(v) + "-" + ",".join(in_idx[k]) for k, v in out_idx.items()
+    }
+    return nodes_group_dict
+
+
+def gen_nodes_group_dict2(edges, nodes_dict):
+    """组合节点传参
+    return:
+        {(source_id, target_id): pass_idx}
+    """
+    out_idx = defaultdict(list)
+    in_idx = defaultdict(list)
+    for edge in edges:
+        # source_id找到对应结点类型
+        out_idx[(edge.info["source_id"], edge.info["target_id"])].append(
+            str(
+                nodes_dict[edge.info["source_id"]].outanchor_to_outidx(
+                    edge.info["source_anchor"]
+                )
+            )
+        )
+        in_idx[(edge.info["source_id"], edge.info["target_id"])].append(
+            str(edge.info["target_anchor"])
+        )
+    nodes_group_dict = {
+        k: ",".join(v) + "-" + ",".join(in_idx[k]) for k, v in out_idx.items()
+    }
+    return nodes_group_dict
+
+
+def cal_score_by_groupby(
+    df,
+    scorer,
+    unit="day",
+    y_ture_col="power",
+    y_pred_col="power_predict",
+    cap=None,
+):
+    """通过groupby计算日月年的单位评分
+
+    Args:
+        df (DataFrame): 要计算的数据
+        scorer (func): 要使用的评价函数
+        unit (str, optional): 评价单位. Defaults to "day".
+        y_ture_col (str, optional): 要使用的真值列. Defaults to "r_apower".
+        y_pred_col (str, optional): 要使用的预测值列. Defaults to "r_apower_predict".
+        cap (num, optional): 场站发电容量. Defaults to None.
+
+    Returns:
+        [type]: 单位和评分值
+    """
+    res = list()
+    dfc = df.copy()
+    if isinstance(dfc.index[0], str):
+        dfc.index = dfc.index.map(pd.to_datetime)
+    if unit == "day":
+        dfc[unit] = pd.to_datetime(dfc.index.strftime("%Y-%m-%d"))
+    elif unit == "month":
+        dfc[unit] = pd.to_datetime(dfc.index.strftime("%Y-%m"))
+    elif unit == "year":
+        dfc[unit] = pd.to_datetime(dfc.index.strftime("%Y-%m"))
+    else:
+        raise ValueError("unit值错误%s" % unit)
+    for i in dfc.groupby(dfc[unit]):
+        if cap:
+            try:
+                score = round(float(scorer(i[1][y_ture_col], i[1][y_pred_col], cap,)), 3,)
+            except:
+                score = round(float(scorer(i[1][y_ture_col], i[1][y_pred_col])), 3)
+        else:
+            score = round(float(scorer(i[1][y_ture_col], i[1][y_pred_col])), 3)
+        res.append([i[0], score])
+    return res

PipeGraphPy/utils/examine.py

@@ -0,0 +1,259 @@
+# coding:utf-8
+"""
+评价函数
+"""
+import numpy as np
+import pandas as pd
+
+
+def rmse(real, pred, cap=49500):
+    s = 1 - np.sqrt(np.mean((real - pred) ** 2))/cap
+    return s*100
+
+
+def rmse1(real, pred, cap=49500):
+    s = np.sqrt(np.mean((real - pred) ** 2))
+    return s
+
+
+def harmonic(real, pred, cap=49500):
+    arr = abs(real / (real+pred) - 0.5) * \
+        abs(real - pred) / (sum(abs(real - pred)))
+    e = 1.0 - 2.0 * arr.sum()
+    return e*100
+
+
+def Racc_AH(real, pred, cap=None):
+    """
+    适用安徽R_acc准确率
+    """
+    deviation = []
+    cal_point = len(real)
+    if len(real[real == 0]) == cal_point:
+        acc_val = 0
+        score_point = cal_point
+    elif len(real[(real > 0) & (real <= 10000)]) == cal_point:
+        acc_val = 0.8
+        score_point = cal_point
+    else:
+        score_point = 0
+        for i in range(len(pred)):
+            if real[i] > 10000:  # 仅大于上限的实测点才参与考核
+                score_point = score_point + 1
+                d = ((real[i] - pred[i]) / real[i])**2
+                deviation.append(d)
+        if len(deviation) != 0:
+            acc_val = 1 - np.sqrt(np.nanmean(deviation))
+        else:
+            acc_val = 1
+
+    return acc_val * 100
+
+
+def Racc_ZJ(real, pred, cap=None):
+    # 浙江R_acc准确率
+    deviation = []
+    for i in range(len(real)):
+        if real[i] <= 0:
+            if pred[i] == 0:
+                deviation.append(0)
+            else:
+                deviation.append(1)
+        else:
+            d = ((pred[i] - real[i]) / real[i])**2
+            if d > 1:
+                d = 1
+            deviation.append(d)
+    acc_val = 1 - np.sqrt(np.nanmean(deviation))
+
+    return acc_val * 100
+
+
+def ACC(real, pred, cap=None):
+    """ACC准确率，适用于华北区域：冀南、冀北、北京、天津"""
+    diff_sum = np.sum(np.abs(real - pred))
+    up = (real - pred) * (real - pred) * np.abs(real - pred) / diff_sum
+    acc_val = 1 - np.sqrt(np.sum(up)) / cap
+
+    return acc_val * 100
+
+
+def AD_acc(real, pred, cap=None):
+    '''
+    AD_acc准确率
+    适用四川电网,江西电网,河南电网,湖北电网,湖南电网,重庆电网光伏
+    '''
+    acc_val = 1 - np.nanmean(np.abs(real - pred)) / cap
+
+    return acc_val * 100
+
+
+def A_corr(real, pred, cap=None):
+    '''
+    A_corr_rate相关性系数，目前仅四川考核
+    '''
+    p_a = np.nanmean(real)
+    pr_a = np.nanmean(pred)
+    acc_val = np.sum(
+        (real - p_a) * (pred - pr_a)) / np.sqrt(
+            np.sum((real - p_a)**2) * np.sum((pred - pr_a)**2))
+    return acc_val * 100
+
+
+def MSP_JS(real, pred, cap=None):
+    '''
+        MSP_jiangsu_rate江苏单点合格率的达标率，即合格率大于等于90%的点占总点的比值
+    '''
+    score_point = 0
+    cal_point = len(real)
+    for i in range(cal_point):
+        acc_val = 1 - np.abs(float(pred[i]) - float(real[i])) / cap
+        if acc_val < 0.9:
+            score_point = score_point + 1
+    acc_val = 1 - float(score_point) / (float(cal_point + 10e-8))
+
+    return acc_val * 100
+
+
+def MSP_FJ(real, pred, cap=None):
+    '''
+        MSP_jiangsu_rate福建单点合格率的达标率，即合格率大于等于75%的点占总点的比值
+    '''
+    score_point = 0
+    cal_point = len(real)
+    for i in range(len(real)):
+        acc_val = 1 - np.abs(float(pred[i]) - float(real[i])) / cap
+        if acc_val < 0.75:
+            score_point = score_point + 1
+    acc_val = 1 - float(score_point) / (float(cal_point + 10e-8))
+
+    return acc_val * 100
+
+
+def SPD(real, pred, cap=None):
+    '''
+        single_diff_Wscore,适用西北风电短期单点绝对偏差SPD考核分的考核，
+        得到的是单位容量考核分
+    '''
+    score_val = 0
+    for i in range(len(real)):
+        # 先计算单点绝对偏差SPD值
+        if (((pred[i] == 0) and (real[i] < 0.03 * cap))
+                or ((real[i] == 0) and (pred[i] < 0.03 * cap))):
+            # 预测为0，实测在3%cap以内或实测为0预测在3%cap以内时，免考核
+            acc_val = 0
+        elif (((pred[i] == 0) and (real[i] >= 0.03 * cap))
+                or ((real[i] == 0) and (pred[i] >= 0.03 * cap))):
+            # 预测为0，实测超出3%cap以外或实测为0预测超出3%cap以外时，全考核
+            acc_val = 1
+        else:
+            acc_val = np.abs((real[i] - pred[i]) / pred[i])
+        # 再计算考核分， score_val_temp为单点考核分
+        if acc_val > 0.25:
+            score_val_temp = (np.abs(pred[i] - real[i]) -
+                              0.25 * pred[i]) / 4 * 0.2 / 10000
+        else:
+            score_val_temp = 0
+        score_val = score_val + score_val_temp
+    return -float(score_val) / cap * 100 * 1000
+
+
+# def rmse(y_true, y_pred, cap=49500):
+#     s = 1 - np.sqrt(np.mean((y_true - y_pred) ** 2))/cap
+#     return s*100
+
+
+def cal_rmse(real, pred, cap=49500):
+    s = np.sqrt(np.mean((real - pred) ** 2))
+    return s
+
+
+def matrix_rmse(real, pred):
+    s = np.sqrt(np.mean((real - np.array(pred).reshape(-1, 1)) ** 2, axis=0))
+    return s
+
+
+def corr(real, pred):
+    from scipy.stats import pearsonr
+    return pearsonr(real, pred)[0]
+
+
+# def harmonic(real, pred, cap=49500):
+#     arr = abs(real / (real+pred) - 0.5) * \
+#         abs(real - pred) / (sum(abs(real - pred)))
+#     e = 1.0 - 2.0 * arr.sum()
+#     return e*100
+
+
+def normalize(values, min, max):
+    qxvalues = (values-min)/(max-min)
+    return qxvalues
+
+
+def rev_normalize(norm_real, max, min):
+    revalue = norm_real * (max - min) + min
+    return revalue
+
+
+def eva_daily_score(self, func, df, cap):
+
+    if hasattr(self, 'ranking_accord'):
+        ranking_accord = self.ranking_accord
+        wfid = self.farm_config['wfid']
+    else:
+        ranking_accord = self.method_config['ranking_accord']
+        wfid = self.method_config['wfid']
+
+    start_dt, end_dt = df.index[0], df.index[-1]
+    if ranking_accord == 'AccumulatedEnergy':
+        obs_source = 'oms'
+        obs_data = self.get_obs_data(wfid=wfid,
+                                     start_dt=start_dt, end_dt=end_dt,
+                                     obs_source=obs_source)
+        obs_data = obs_data.rename(columns={'rectime': 'ptime'})
+        df['ptime'] = df.index
+        df = pd.merge(df, obs_data, how='inner', on='ptime')
+        df.index = df.ptime
+        del df['ptime']
+
+        data = []
+        for day, gdf in df.groupby(by=lambda x: x.strftime('%Y-%m-%d')):
+            if gdf.shape[0] < 10:
+                continue
+            if self.ranking_accord == 'AccumulatedEnergy':
+                if gdf.shape[1] == 5:
+                    score = func(gdf.power, gdf.predict, cap)
+                else:
+                    score = func(gdf.power_y, gdf.predict_power_x, cap)
+            else:
+                score = func(gdf.iloc[:, 0], gdf.iloc[:, 1], cap)
+            data.append([day, score])
+    else:
+        data = []
+        for day, gdf in df.groupby(by=lambda x: x.strftime('%Y-%m-%d')):
+            if gdf.shape[0] < 70:
+                continue
+            score = func(gdf.iloc[:, 0], gdf.iloc[:, 1], cap)
+            data.append([day, score])
+
+    return pd.DataFrame(
+        data, columns=['day', 'accurate']).set_index(keys='day')
+
+
+def AccumulatedEnergy(real, pred, cap=49500):
+    data = pd.DataFrame()
+    data['real'] = real
+    data['pred'] = pred
+    data['error'] = np.abs((data['real']-data['pred']))/data['pred']
+    data.ix[(data['real'] == 0) & (data['pred'] <= 0.03*cap), 'error'] = 0
+    data.ix[(data['real'] == 0) & (data['pred'] >= 0.03*cap), 'error'] = 1
+    data.ix[(data['real'] <= 0.03*cap) & (data['pred'] == 0), 'error'] = 0
+    data.ix[(data['real'] > 0.03*cap) & (data['pred'] == 0), 'error'] = 1
+    data = data.reset_index(drop=True)
+    data['fenshu'] = 0
+    for nn in range(0, len(data)):
+        if data.ix[nn, 'error'] > 0.2:
+            data.ix[nn, 'fenshu'] = np.abs((np.abs(
+                data.ix[nn, 'real']-data.ix[nn, 'pred']
+                )-0.2*data.ix[nn, 'pred']))/10000*0.25*0.2
+    return -np.sum(data['fenshu'])

PipeGraphPy/utils/file_operate.py

@@ -0,0 +1,101 @@
+import pickle
+import traceback
+import shutil
+import joblib
+import os
+from datetime import datetime, timedelta
+from PipeGraphPy.db.models import PredictRecordTB
+
+
+def joblib_dumps(file_path, data):
+    """joblib文件的保存"""
+    folder_path = os.path.dirname(file_path)
+    if not os.path.exists(folder_path):
+        os.makedirs(folder_path)
+    joblib.dump(data, file_path)
+
+
+def joblib_loads(file_path):
+    """载入joblib模型文件"""
+    if not os.path.isfile(file_path):
+        raise FileNotFoundError("模型文件不存在: %s" % file_path)
+    try:
+        data = joblib.load(file_path)
+        return data
+    except Exception as e:
+        raise Exception(f"载入joblib文件失败:{file_path}, 请检查文件的正确性")
+
+
+def pickle_dumps(file_path, data):
+    """pickle文件的保存"""
+    if os.path.isfile(file_path):
+        os.remove(file_path)
+    folder_path = os.path.dirname(file_path)
+    if not os.path.exists(folder_path):
+        os.makedirs(folder_path)
+    with open(file_path, "wb") as f:
+        pickle.dump(data, f)
+
+
+def pickle_loads(file_path):
+    """载入pickle文件"""
+    if not os.path.isfile(file_path):
+        raise FileNotFoundError("pickle文件不存在: %s" % file_path)
+    try:
+        with open(file_path, "rb") as f:
+            data = pickle.load(f)
+            return data
+    except Exception as e:
+        error = traceback.format_exc()
+        raise Exception(f"载入pickle文件失败:{file_path}, 请检查文件的正确性\n{error}")
+
+
+def get_file_path(root_path):
+    """获取该目录下所有的文件名称和目录名称"""
+    dir_list, file_list = [], []
+
+    def add_path(dir):
+        nonlocal dir_list
+        nonlocal file_list
+        for dir_file in os.listdir(dir):
+            # 删除七天以前的预测记录
+            del_date = ((datetime.utcnow()+timedelta(hours=8)) - timedelta(days=7)).strftime("%Y-%m-%d")
+            PredictRecordTB.rm(ctime=("<", del_date))
+            # 删除之前的缓存气象目录
+            dir_file_path = os.path.join(dir, dir_file)
+            if os.path.isdir(dir_file_path):
+                dir_list.append(dir_file_path)
+                add_path(dir_file_path)
+            else:
+                file_list.append(dir_file_path)
+
+    add_path(root_path)
+    return dir_list, file_list
+
+
+def get_file_path_walk(root_path):
+    """获取该目录下所有的文件名称和目录名称"""
+    dir_list, file_list = [], []
+    for root, dirs, files in os.walk(root_path):
+        for dir in dirs:
+            dir_list.append(os.path.join(root, dir))
+        for file in files:
+            file_list.append(os.path.join(root, file))
+    return dir_list, file_list
+
+
+def delete_filepath(paths):
+    """删除路径或文件列表
+
+    Args:
+        paths (list): 路径或文件列表
+    """
+    assert isinstance(paths, list), TypeError("paths必须是列表")
+    for p in paths:
+        if os.path.isdir(p):
+            shutil.rmtree(p)
+        elif os.path.isfile(p):
+            os.remove(p)
+        else:
+            pass
+