PipeGraphPy 1.0.18__py3-none-manylinux1_x86_64.whl

PipeGraphPy/__init__.py

@@ -0,0 +1,10 @@
+# coding:utf-8
+
+__version__ = "1.0.18"
+
+from PipeGraphPy.core.graph import Graph, graph_predict, graph_evaluate, online_graph_evaluate, online_graph_predict, graph_run, graph_backtest
+from PipeGraphPy.core.module import Module
+from PipeGraphPy.core.node import Node
+
+__all__ = ['Node', 'Module', 'Graph', 'graph_predict', 'graph_evaluate',
+        'online_graph_evaluate', 'online_graph_predict', 'graph_run', 'graph_backtest']

PipeGraphPy/common.py

@@ -0,0 +1,4 @@
+# 本文件存储通用变量或全局变量
+
+
+multi_graph = dict()

PipeGraphPy/config/__init__.py

@@ -0,0 +1,277 @@
+# coding:utf-8
+
+"""
+Settings and configuration for PGP.
+
+Read values from the module specified by the PGP_SETTINGS_MODULE environment
+variable, and then from PipeGraphPy.conf.default_settings; see the default_settings.py
+for a list of all possible variables.
+
+"""
+import importlib
+import os
+import traceback
+import PipeGraphPy
+from PipeGraphPy.config import default_settings
+from PipeGraphPy.exceptions import ImproperlyConfigured
+from PipeGraphPy.utils.functional import LazyObject, empty
+from dbpoolpy import settings as dbsettings
+
+os.environ['PGP_SETTINGS_MODULE'] = 'PipeGraphPy.config.custom'
+
+ENVIRONMENT_VARIABLE = "PGP_SETTINGS_MODULE"
+
+PASSWORD_RESET_TIMEOUT_DAYS_DEPRECATED_MSG = (
+    'The PASSWORD_RESET_TIMEOUT_DAYS setting is deprecated. Use '
+    'PASSWORD_RESET_TIMEOUT instead.'
+)
+
+DEFAULT_HASHING_ALGORITHM_DEPRECATED_MSG = (
+    'The DEFAULT_HASHING_ALGORITHM transitional setting is deprecated. '
+    'Support for it and tokens, cookies, sessions, and signatures that use '
+    'SHA-1 hashing algorithm will be removed in PGP 4.0.'
+)
+
+
+class SettingsReference(str):
+    """
+    String subclass which references a current settings value. It's treated as
+    the value in memory but serializes to a settings.NAME attribute reference.
+    """
+
+    def __new__(self, value, setting_name):
+        return str.__new__(self, value)
+
+    def __init__(self, value, setting_name):
+        self.setting_name = setting_name
+
+
+class LazySettings(LazyObject):
+    """
+    A lazy proxy for either global PGP settings or a custom settings object.
+    The user can manually configure settings prior to using them. Otherwise,
+    PGP uses the settings module pointed to by PGP_SETTINGS_MODULE.
+    """
+
+    def _setup(self, name=None):
+        """
+        Load the settings module pointed to by the environment variable. This
+        is used the first time settings are needed, if the user hasn't
+        configured settings manually.
+        """
+        settings_module = os.environ.get(ENVIRONMENT_VARIABLE)
+        if not settings_module:
+            desc = ("setting %s" % name) if name else "settings"
+            raise ImproperlyConfigured(
+                "Requested %s, but settings are not configured. "
+                "You must either define the environment variable %s "
+                "or call settings.configure() before accessing settings."
+                % (desc, ENVIRONMENT_VARIABLE))
+
+        self._wrapped = Settings(settings_module)
+
+    def __repr__(self):
+        # Hardcode the class name as otherwise it yields 'Settings'.
+        if self._wrapped is empty:
+            return '<LazySettings [Unevaluated]>'
+        return '<LazySettings "%(settings_module)s">' % {
+            'settings_module': self._wrapped.SETTINGS_MODULE,
+        }
+
+    def __getattr__(self, name):
+        """Return the value of a setting and cache it in self.__dict__."""
+        if self._wrapped is empty:
+            self._setup(name)
+        val = getattr(self._wrapped, name)
+
+        # Special case some settings which require further modification.
+        # This is done here for performance reasons so the
+        #  modified value is cached.
+
+        # if name in {'MEDIA_URL', 'STATIC_URL'} and val is not None:
+        #     val = self._add_script_prefix(val)
+        # elif name == 'SECRET_KEY' and not val:
+        #     raise ImproperlyConfigured(
+        #         "The SECRET_KEY setting must not be empty.")
+
+        self.__dict__[name] = val
+        return val
+
+    def __setattr__(self, name, value):
+        """
+        Set the value of setting. Clear all cached values if _wrapped changes
+        (@override_settings does this) or clear single values when set.
+        """
+        if name == '_wrapped':
+            self.__dict__.clear()
+        else:
+            self.__dict__.pop(name, None)
+        if name == 'DEBUG':
+            dbsettings.DEBUG = value
+        super().__setattr__(name, value)
+
+    def __delattr__(self, name):
+        """Delete a setting and clear it from cache if needed."""
+        super().__delattr__(name)
+        self.__dict__.pop(name, None)
+
+    def configure(self, default_settings=default_settings, **options):
+        """
+        Called to manually configure the settings. The 'default_settings'
+        parameter sets where to retrieve any unspecified values from (its
+        argument must support attribute access (__getattr__)).
+        """
+        if self._wrapped is not empty:
+            raise RuntimeError('Settings already configured.')
+        holder = UserSettingsHolder(default_settings)
+        for name, value in options.items():
+            if not name.isupper():
+                raise TypeError('Setting %r must be uppercase.' % name)
+            setattr(holder, name, value)
+        self._wrapped = holder
+
+    # @staticmethod
+    # def _add_script_prefix(value):
+    #     """
+    #     Add SCRIPT_NAME prefix to relative paths.
+
+    #     Useful when the app is being served at a subpath and manually prefixing
+    #     subpath to STATIC_URL and MEDIA_URL in settings is inconvenient.
+    #     """
+    #     # Don't apply prefix to absolute paths and URLs.
+    #     if value.startswith(('http://', 'https://', '/')):
+    #         return value
+    #     from PipeGraphPy.urls import get_script_prefix
+    #     return '%s%s' % (get_script_prefix(), value)
+
+    @property
+    def configured(self):
+        """Return True if the settings have already been configured."""
+        return self._wrapped is not empty
+
+    @property
+    def PASSWORD_RESET_TIMEOUT_DAYS(self):
+        stack = traceback.extract_stack()
+        # Show a warning if the setting is used outside of PGP.
+        # Stack index: -1 this line, -2 the caller.
+        filename, _, _, _ = stack[-2]
+        if not filename.startswith(os.path.dirname(PipeGraphPy.__file__)):
+            pass
+            #  warnings.warn(
+            #  PASSWORD_RESET_TIMEOUT_DAYS_DEPRECATED_MSG,
+            #  RemovedInPGP40Warning,
+            #  stacklevel=2,
+            #  )
+        return self.__getattr__('PASSWORD_RESET_TIMEOUT_DAYS')
+
+
+class Settings:
+    def __init__(self, settings_module):
+        # update this dict from global
+        # settings (but only for ALL_CAPS settings)
+        for setting in dir(default_settings):
+            if setting.isupper():
+                setattr(self, setting, getattr(default_settings, setting))
+
+        # store the settings module in case someone later cares
+        self.SETTINGS_MODULE = settings_module
+
+        mod = importlib.import_module(self.SETTINGS_MODULE)
+
+        tuple_settings = (
+            "INSTALLED_APPS",
+            "TEMPLATE_DIRS",
+            "LOCALE_PATHS",
+        )
+        self._explicit_settings = set()
+        for setting in dir(mod):
+            if setting.isupper():
+                setting_value = getattr(mod, setting)
+
+                if (setting in tuple_settings and
+                        not isinstance(setting_value, (list, tuple))):
+                    raise ImproperlyConfigured(
+                        "The %s setting must be a list or a tuple. " % setting)
+                setattr(self, setting, setting_value)
+                self._explicit_settings.add(setting)
+
+        if self.is_overridden('PASSWORD_RESET_TIMEOUT_DAYS'):
+            if self.is_overridden('PASSWORD_RESET_TIMEOUT'):
+                raise ImproperlyConfigured(
+                    'PASSWORD_RESET_TIMEOUT_DAYS/PASSWORD_RESET_TIMEOUT are '
+                    'mutually exclusive.'
+                )
+            setattr(self, 'PASSWORD_RESET_TIMEOUT',
+                    self.PASSWORD_RESET_TIMEOUT_DAYS * 60 * 60 * 24)
+            # warnings.warn(PASSWORD_RESET_TIMEOUT_DAYS_DEPRECATED_MSG,
+            #               RemovedInPGP40Warning)
+
+        # if self.is_overridden('DEFAULT_HASHING_ALGORITHM'):
+        #     warnings.warn(DEFAULT_HASHING_ALGORITHM_DEPRECATED_MSG,
+        #                   RemovedInPGP40Warning)
+
+    def is_overridden(self, setting):
+        return setting in self._explicit_settings
+
+    def __repr__(self):
+        return '<%(cls)s "%(settings_module)s">' % {
+            'cls': self.__class__.__name__,
+            'settings_module': self.SETTINGS_MODULE,
+        }
+
+
+class UserSettingsHolder:
+    """Holder for user configured settings."""
+    # SETTINGS_MODULE doesn't make much sense in the manually configured
+    # (standalone) case.
+    SETTINGS_MODULE = None
+
+    def __init__(self, default_settings):
+        """
+        Requests for configuration variables not in this class are satisfied
+        from the module specified in default_settings (if possible).
+        """
+        self.__dict__['_deleted'] = set()
+        self.default_settings = default_settings
+
+    def __getattr__(self, name):
+        if not name.isupper() or name in self._deleted:
+            raise AttributeError
+        return getattr(self.default_settings, name)
+
+    def __setattr__(self, name, value):
+        self._deleted.discard(name)
+        # if name == 'PASSWORD_RESET_TIMEOUT_DAYS':
+        #     setattr(self, 'PASSWORD_RESET_TIMEOUT', value * 60 * 60 * 24)
+        #     warnings.warn(PASSWORD_RESET_TIMEOUT_DAYS_DEPRECATED_MSG,
+        #                   RemovedInPGP40Warning)
+        # if name == 'DEFAULT_HASHING_ALGORITHM':
+        #     warnings.warn(DEFAULT_HASHING_ALGORITHM_DEPRECATED_MSG,
+        #                   RemovedInPGP40Warning)
+        super().__setattr__(name, value)
+
+    def __delattr__(self, name):
+        self._deleted.add(name)
+        if hasattr(self, name):
+            super().__delattr__(name)
+
+    def __dir__(self):
+        return sorted(
+            s for s in [*self.__dict__, *dir(self.default_settings)]
+            if s not in self._deleted
+        )
+
+    def is_overridden(self, setting):
+        deleted = (setting in self._deleted)
+        set_locally = (setting in self.__dict__)
+        set_on_default = getattr(
+            self.default_settings, 'is_overridden', lambda s: False)(setting)
+        return deleted or set_locally or set_on_default
+
+    def __repr__(self):
+        return '<%(cls)s>' % {
+            'cls': self.__class__.__name__,
+        }
+
+
+settings = LazySettings()

PipeGraphPy/config/config_light.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+
+"""This file is part of the TPOT library.
+TPOT was primarily developed at the University of Pennsylvania by:
+    - Randal S. Olson (rso@randalolson.com)
+    - Weixuan Fu (weixuanf@upenn.edu)
+    - Daniel Angell (dpa34@drexel.edu)
+    - and many more generous open source contributors
+TPOT is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as
+published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
+TPOT is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Lesser General Public License for more details.
+You should have received a copy of the GNU Lesser General Public
+License along with TPOT. If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import numpy as np
+
+# Check the TPOT documentation for information on the structure of config dicts
+
+regressor_config_dict_light = {
+
+    'sklearn.linear_model.ElasticNetCV': {
+        'l1_ratio': np.arange(0.0, 1.01, 0.05),
+        'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1]
+    },
+
+    'sklearn.tree.DecisionTreeRegressor': {
+        'max_depth': range(1, 11),
+        'min_samples_split': range(2, 21),
+        'min_samples_leaf': range(1, 21)
+    },
+
+    'sklearn.neighbors.KNeighborsRegressor': {
+        'n_neighbors': range(1, 101),
+        'weights': ["uniform", "distance"],
+        'p': [1, 2]
+    },
+
+    'sklearn.linear_model.LassoLarsCV': {
+        'normalize': [True, False]
+    },
+
+    'sklearn.svm.LinearSVR': {
+        'loss': ["epsilon_insensitive", "squared_epsilon_insensitive"],
+        'dual': [True, False],
+        'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1],
+        'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.],
+        'epsilon': [1e-4, 1e-3, 1e-2, 1e-1, 1.]
+    },
+
+    'sklearn.linear_model.RidgeCV': {
+    },
+
+
+    # Preprocesssors
+    'sklearn.preprocessing.Binarizer': {
+        'threshold': np.arange(0.0, 1.01, 0.05)
+    },
+
+
+    'sklearn.cluster.FeatureAgglomeration': {
+        'linkage': ['ward', 'complete', 'average'],
+        'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine']
+    },
+
+    'sklearn.preprocessing.MaxAbsScaler': {
+    },
+
+    'sklearn.preprocessing.MinMaxScaler': {
+    },
+
+    'sklearn.preprocessing.Normalizer': {
+        'norm': ['l1', 'l2', 'max']
+    },
+
+    'sklearn.kernel_approximation.Nystroem': {
+        'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'],
+        'gamma': np.arange(0.0, 1.01, 0.05),
+        'n_components': range(1, 11)
+    },
+
+    'sklearn.decomposition.PCA': {
+        'svd_solver': ['randomized'],
+        'iterated_power': range(1, 11)
+    },
+
+    'sklearn.kernel_approximation.RBFSampler': {
+        'gamma': np.arange(0.0, 1.01, 0.05)
+    },
+
+    'sklearn.preprocessing.RobustScaler': {
+    },
+
+    'sklearn.preprocessing.StandardScaler': {
+    },
+
+
+    # Selectors
+    'sklearn.feature_selection.SelectFwe': {
+        'alpha': np.arange(0, 0.05, 0.001),
+        'score_func': {
+            'sklearn.feature_selection.f_regression': None
+        }
+    },
+
+    'sklearn.feature_selection.SelectPercentile': {
+        'percentile': range(1, 100),
+        'score_func': {
+            'sklearn.feature_selection.f_regression': None
+        }
+    },
+
+    'sklearn.feature_selection.VarianceThreshold': {
+        'threshold': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
+    }
+
+}

PipeGraphPy/config/custom.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+
+
+

PipeGraphPy/config/default_settings.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Default PGP settings.
+"""
+import os
+import sys
+# from pyecharts.globals import NotebookType
+
+class PGP_ENV:
+    DEV = "dev"   # 本地开发
+    DEBUG = "debug" # 本地测试
+    RPC = "rpc"   # rpc
+    RUN_WORKER = "run_worker"  # 训练worker
+    PREDICT_WORKER = "predict_worker"   # 预测worker
+    SDK = "sdk"  # sdk
+    NONE = "none"   # 非执行训练和预测环境
+
+
+DEBUG = False if str(os.environ.get("PGP_NOT_DEBUG")) == "1" else True
+ISDEV = False if str(os.environ.get("PGP_NOT_DEV")) == "1" else True
+ISJUPYTER = True if str(os.environ.get("PGP_JUPYTER")) == "1" else False
+RUN_PERMISSION = True if str(os.environ.get("PGP_RUN_PERMISSION")) == "1" else False
+RUN_MODEL_SAVE_PATH = os.environ.get("PGP_RUN_MODEL_SAVE_PATH", "")
+SDK_MODEL_SAVE_PATH = os.environ.get("PGP_SDK_MODEL_SAVE_PATH", "")
+PREDICT_MODEL_SAVE_PATH = os.environ.get("PGP_PREDICT_MODEL_SAVE_PATH", '')
+PREDICT_RESULT_SAVE_PATH = os.environ.get("PGP_PREDICT_RESULT_SAVE_PATH", '')
+BACKTEST_RESULT_SAVE_PATH = os.environ.get("PGP_BACKTEST_RESULT_SAVE_PATH", '')
+RUN_RESULT_SAVE_PATH = os.environ.get("PGP_RUN_RESULT_SAVE_PATH", '')
+EVALUATE_RESULT_SAVE_PATH = os.environ.get("PGP_EVALUATE_RESULT_SAVE_PATH", '')
+TEMP_SAVE_PATH = os.environ.get("PGP_TEMP_SAVE_PATH", "")
+CUSTOM_CACHE_PATH = os.environ.get("PGP_CUSTOM_CACHE_PATH", '')
+REDIS_HOST = os.environ.get("REDIS_HOST", '39.105.185.60')
+REDIS_PORT = int(os.environ.get("REDIS_PORT", '6371'))
+PGP_DB_HOST = os.environ.get("PGP_DB_HOST", '39.105.185.60')
+PGP_DB_PORT = int(os.environ.get("PGP_DB_PORT", '33061'))
+AMQP_URL = os.environ.get("PGP_AMQP_URL", 'amqp://mlf:mlfadmin@47.94.251.175:6296/mlf_vhost')
+RUN_TIMEOUT = int(os.environ.get("RUN_TIMEOUT", 60 * 60 * 6))   # 训练超时2小时
+PREDICT_TIMEOUT = int(os.environ.get("PREDICT_TIMEOUT", 60 * 10))   # 预测超时5分钟
+RT_DB_CONNECTOR = os.environ.get("RT_DB_CONNECTOR", '')
+ISONLINE = True if str(os.environ.get("PGP_IS_ONLINE")) == "1" else False
+DEPLOYMENT_ENV = os.environ.get("PGP_DEPLOYMENT_ENV", "none")
+RUN_ENV = os.environ.get("PGP_RUN_ENV", 'test')    # 运行环境, test, online, dev, sdk, image
+DBPOOL_SERVER_NAME = os.environ.get("PGP_DBPOOL_SERVER_NAME", 'mlf')
+PGP_PDB_PORT = int(os.environ.get("PGP_PDB_PORT", '6688'))
+
+# 模型保存配置
+if DEPLOYMENT_ENV == PGP_ENV.SDK:
+    HOME_CACHE_PATH = os.path.expanduser('~/.cache/PipeGraphPy/sdk/')
+    if not SDK_MODEL_SAVE_PATH:
+        SDK_MODEL_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "sdk")
+    if not os.path.exists(SDK_MODEL_SAVE_PATH):
+        os.makedirs(SDK_MODEL_SAVE_PATH)
+else:
+    HOME_CACHE_PATH = os.path.expanduser('~/.cache/PipeGraphPy/')
+if not RUN_MODEL_SAVE_PATH:
+    RUN_MODEL_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "models")
+if not PREDICT_MODEL_SAVE_PATH:
+    PREDICT_MODEL_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "models")
+if not PREDICT_RESULT_SAVE_PATH:
+    PREDICT_RESULT_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "predict_result")
+if not BACKTEST_RESULT_SAVE_PATH:
+    BACKTEST_RESULT_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "backtest_result")
+if not RUN_RESULT_SAVE_PATH:
+    RUN_RESULT_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "run_result")
+if not EVALUATE_RESULT_SAVE_PATH:
+    EVALUATE_RESULT_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "evaluate_result")
+if not TEMP_SAVE_PATH:
+    TEMP_SAVE_PATH = os.path.join(HOME_CACHE_PATH, "temp")
+if not CUSTOM_CACHE_PATH:
+    CUSTOM_CACHE_PATH = os.path.join(HOME_CACHE_PATH, "custom")
+if not os.path.exists(CUSTOM_CACHE_PATH):
+    os.makedirs(CUSTOM_CACHE_PATH)
+    _init_file = os.path.join(CUSTOM_CACHE_PATH, "__init__.py")
+    with open(_init_file, "w") as f:
+        f.write("#")
+sys.path.append(os.path.dirname(CUSTOM_CACHE_PATH))
+MODEL_SAVE_NAME = "graph_model_{graph_id}"
+ALGO_MODEL_SAVE_NAME = "algo_model_{graph_id}_{node_id}_{algo_mod_type}_{idx}"
+
+# 运行模式 1: 包引用 2：grpc
+RUN_PATTERN = 1
+
+# redis设置
+REDIS_DB = 3 if str(os.environ.get("PGP_NOT_DEV")) == "1" else 4
+REDIS_KEY_TTL = 7 * 24 * 60 * 60  # 过期时间,默认7天
+
+# celery 存储设置
+CELERY_REDIS_HOST = None
+CELERY_REDIS_PORT = None
+CELERY_REDIS_DB = None
+
+
+# 时间格式
+DATETIME_FORMAT = "%H:%M:%S"
+DATETIME_TOTAL_FORMAT = "%Y-%m-%d %H:%M:%S"
+DATETIME_INPUT_FORMATS = [
+    "%Y-%m-%d %H:%M:%S",
+]
+
+# 计算存储引擎
+STORAGE_ENGINE = "file"
+
+# 结构数据库配置
+DATABASES = {
+    "ENGINE": "sqlite3",
+    "NAME": "PipeGraphPy",
+}
+
+# 数据库配置
+DATABASES_POOL = None
+# 数据库名称
+DATABASE_NAME = None
+DATABASE_SERVER_NAME = None
+DATABASE_DB_NAME = None
+
+
+# jupyter_type
+# JUPYTER_TYPE = NotebookType.JUPYTER_LAB
+
+
+# SDK是否显示log日志
+SDK_SHOW_LOG = False
+