PipeGraphPy 2.0.6__py3-none-win_amd64.whl

PipeGraphPy/utils/functional.py

@@ -0,0 +1,422 @@
+import copy
+import itertools
+import operator
+from functools import total_ordering, wraps
+
+
+class cached_property:
+    """
+    Decorator that converts a method with a single self argument into a
+    property cached on the instance.
+
+    A cached property can be made out of an existing method:
+    (e.g. ``url = cached_property(get_absolute_url)``).
+    The optional ``name`` argument is obsolete as of Python 3.6 and will be
+    deprecated in PGP 4.0 (#30127).
+    """
+    name = None
+
+    @staticmethod
+    def func(instance):
+        raise TypeError(
+            'Cannot use cached_property instance without calling '
+            '__set_name__() on it.'
+        )
+
+    def __init__(self, func, name=None):
+        self.real_func = func
+        self.__doc__ = getattr(func, '__doc__')
+
+    def __set_name__(self, owner, name):
+        if self.name is None:
+            self.name = name
+            self.func = self.real_func
+        elif name != self.name:
+            raise TypeError(
+                "Cannot assign the same cached_property to two different names "
+                "(%r and %r)." % (self.name, name)
+            )
+
+    def __get__(self, instance, cls=None):
+        """
+        调用函数并将返回值放入实例中。__dict__，以便对实例的后续属性访问返回缓存的值，
+        而不是调用cached_property.__get__()。
+        """
+        if instance is None:
+            return self
+        res = instance.__dict__[self.name] = self.func(instance)
+        return res
+
+
+class classproperty:
+    """
+    Decorator that converts a method with a single cls argument into a property
+    that can be accessed directly from the class.
+    """
+
+    def __init__(self, method=None):
+        self.fget = method
+
+    def __get__(self, instance, cls=None):
+        return self.fget(cls)
+
+    def getter(self, method):
+        self.fget = method
+        return self
+
+
+class Promise:
+    """
+    Base class for the proxy class created in the closure of the lazy function.
+    It's used to recognize promises in code.
+    """
+    pass
+
+
+def lazy(func, *resultclasses):
+    """
+    Turn any callable into a lazy evaluated callable. result classes or types
+    is required -- at least one is needed so that the automatic forcing of
+    the lazy evaluation code is triggered. Results are not memoized; the
+    function is evaluated on every access.
+    """
+
+    @total_ordering
+    class __proxy__(Promise):
+        """
+        封装一个函数调用，并充当调用该函数结果的方法的代理。
+        在调用结果上的一个方法之前，函数不会被求值。
+        """
+        __prepared = False
+
+        def __init__(self, args, kw):
+            self.__args = args
+            self.__kw = kw
+            if not self.__prepared:
+                self.__prepare_class__()
+            self.__class__.__prepared = True
+
+        def __reduce__(self):
+            return (
+                _lazy_proxy_unpickle,
+                (func, self.__args, self.__kw) + resultclasses
+            )
+
+        def __repr__(self):
+            return repr(self.__cast())
+
+        @classmethod
+        def __prepare_class__(cls):
+            for resultclass in resultclasses:
+                for type_ in resultclass.mro():
+                    for method_name in type_.__dict__:
+                        # All __promise__ return the same wrapper method, they
+                        # look up the correct implementation when called.
+                        if hasattr(cls, method_name):
+                            continue
+                        meth = cls.__promise__(method_name)
+                        setattr(cls, method_name, meth)
+            cls._delegate_bytes = bytes in resultclasses
+            cls._delegate_text = str in resultclasses
+            assert not (cls._delegate_bytes and cls._delegate_text), (
+                "Cannot call lazy() with both bytes and text return types.")
+            if cls._delegate_text:
+                cls.__str__ = cls.__text_cast
+            elif cls._delegate_bytes:
+                cls.__bytes__ = cls.__bytes_cast
+
+        @classmethod
+        def __promise__(cls, method_name):
+            # Builds a wrapper around some magic method
+            def __wrapper__(self, *args, **kw):
+                # Automatically triggers the evaluation of a lazy value and
+                # applies the given magic method of the result type.
+                res = func(*self.__args, **self.__kw)
+                return getattr(res, method_name)(*args, **kw)
+            return __wrapper__
+
+        def __text_cast(self):
+            return func(*self.__args, **self.__kw)
+
+        def __bytes_cast(self):
+            return bytes(func(*self.__args, **self.__kw))
+
+        def __bytes_cast_encoded(self):
+            return func(*self.__args, **self.__kw).encode()
+
+        def __cast(self):
+            if self._delegate_bytes:
+                return self.__bytes_cast()
+            elif self._delegate_text:
+                return self.__text_cast()
+            else:
+                return func(*self.__args, **self.__kw)
+
+        def __str__(self):
+            # object defines __str__(), so __prepare_class__() won't overload
+            # a __str__() method from the proxied class.
+            return str(self.__cast())
+
+        def __eq__(self, other):
+            if isinstance(other, Promise):
+                other = other.__cast()
+            return self.__cast() == other
+
+        def __lt__(self, other):
+            if isinstance(other, Promise):
+                other = other.__cast()
+            return self.__cast() < other
+
+        def __hash__(self):
+            return hash(self.__cast())
+
+        def __mod__(self, rhs):
+            if self._delegate_text:
+                return str(self) % rhs
+            return self.__cast() % rhs
+
+        def __add__(self, other):
+            return self.__cast() + other
+
+        def __radd__(self, other):
+            return other + self.__cast()
+
+        def __deepcopy__(self, memo):
+            # Instances of this class are effectively immutable. It's just a
+            # collection of functions. So we don't need to do anything
+            # complicated for copying.
+            memo[id(self)] = self
+            return self
+
+    @wraps(func)
+    def __wrapper__(*args, **kw):
+        # Creates the proxy object, instead of the actual value.
+        return __proxy__(args, kw)
+
+    return __wrapper__
+
+
+def _lazy_proxy_unpickle(func, args, kwargs, *resultclasses):
+    return lazy(func, *resultclasses)(*args, **kwargs)
+
+
+def lazystr(text):
+    """
+    Shortcut for the common case of a lazy callable that returns str.
+    """
+    return lazy(str, str)(text)
+
+
+def keep_lazy(*resultclasses):
+    """
+    A decorator that allows a function to be called with one or more lazy
+    arguments. If none of the args are lazy, the function is evaluated
+    immediately, otherwise a __proxy__ is returned that will evaluate the
+    function when needed.
+    """
+    if not resultclasses:
+        raise TypeError("You must pass at least one argument to keep_lazy().")
+
+    def decorator(func):
+        lazy_func = lazy(func, *resultclasses)
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if any(isinstance(arg, Promise) for arg in itertools.chain(args, kwargs.values())):
+                return lazy_func(*args, **kwargs)
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
+def keep_lazy_text(func):
+    """
+    A decorator for functions that accept lazy arguments and return text.
+    """
+    return keep_lazy(str)(func)
+
+
+empty = object()
+
+
+def new_method_proxy(func):
+    def inner(self, *args):
+        if self._wrapped is empty:
+            self._setup()
+        return func(self._wrapped, *args)
+    return inner
+
+
+class LazyObject:
+    """
+    一个类的包装器，可用于延迟包装类的实例化。
+
+    通过子类化，您有机会截取和修改实例化。如果你不需要这样做，使用SimpleLazyObject。
+
+    """
+
+    # 在跟踪__init__(#19456)时避免无限递归
+    _wrapped = None
+
+    def __init__(self):
+        # 注意:如果子类重写了__init__()，它可能需要重写
+        # override __copy__()和__deepcopy__()。
+        self._wrapped = empty
+
+    __getattr__ = new_method_proxy(getattr)
+
+    def __setattr__(self, name, value):
+        if name == "_wrapped":
+            # 赋值__dict__以避免无限__setattr__循环。
+            self.__dict__["_wrapped"] = value
+        else:
+            if self._wrapped is empty:
+                self._setup()
+            setattr(self._wrapped, name, value)
+
+    def __delattr__(self, name):
+        if name == "_wrapped":
+            raise TypeError("can't delete _wrapped.")
+        if self._wrapped is empty:
+            self._setup()
+        delattr(self._wrapped, name)
+
+    def _setup(self):
+        """
+        Must be implemented by subclasses to initialize the wrapped object.
+        """
+        raise NotImplementedError(
+            'subclasses of LazyObject must provide a _setup() method')
+
+    # Because we have messed with __class__ below, we confuse pickle as to what
+    # class we are pickling. We're going to have to initialize the wrapped
+    # object to successfully pickle it, so we might as well just pickle the
+    # wrapped object since they're supposed to act the same way.
+    #
+    # Unfortunately, if we try to simply act like the wrapped object, the ruse
+    # will break down when pickle gets our id(). Thus we end up with pickle
+    # thinking, in effect, that we are a distinct object from the wrapped
+    # object, but with the same __dict__. This can cause problems (see #25389).
+    #
+    # So instead, we define our own __reduce__ method and custom unpickler. We
+    # pickle the wrapped object as the unpickler's argument, so that pickle
+    # will pickle it normally, and then the unpickler simply returns its
+    # argument.
+    def __reduce__(self):
+        if self._wrapped is empty:
+            self._setup()
+        return (unpickle_lazyobject, (self._wrapped,))
+
+    def __copy__(self):
+        if self._wrapped is empty:
+            # If uninitialized, copy the wrapper. Use type(self), not
+            # self.__class__, because the latter is proxied.
+            return type(self)()
+        else:
+            # If initialized, return a copy of the wrapped object.
+            return copy.copy(self._wrapped)
+
+    def __deepcopy__(self, memo):
+        if self._wrapped is empty:
+            # We have to use type(self), not self.__class__, because the
+            # latter is proxied.
+            result = type(self)()
+            memo[id(self)] = result
+            return result
+        return copy.deepcopy(self._wrapped, memo)
+
+    __bytes__ = new_method_proxy(bytes)
+    __str__ = new_method_proxy(str)
+    __bool__ = new_method_proxy(bool)
+
+    # Introspection support
+    __dir__ = new_method_proxy(dir)
+
+    # Need to pretend to be the wrapped class, for the sake of objects that
+    # care about this (especially in equality tests)
+    __class__ = property(new_method_proxy(operator.attrgetter("__class__")))
+    __eq__ = new_method_proxy(operator.eq)
+    __lt__ = new_method_proxy(operator.lt)
+    __gt__ = new_method_proxy(operator.gt)
+    __ne__ = new_method_proxy(operator.ne)
+    __hash__ = new_method_proxy(hash)
+
+    # List/Tuple/Dictionary methods support
+    __getitem__ = new_method_proxy(operator.getitem)
+    __setitem__ = new_method_proxy(operator.setitem)
+    __delitem__ = new_method_proxy(operator.delitem)
+    __iter__ = new_method_proxy(iter)
+    __len__ = new_method_proxy(len)
+    __contains__ = new_method_proxy(operator.contains)
+
+
+def unpickle_lazyobject(wrapped):
+    """
+    用于unpickle懒惰对象。只需返回它的包装的对象的参数。
+    """
+    return wrapped
+
+
+class SimpleLazyObject(LazyObject):
+    """
+    A lazy object initialized from any function.
+
+    Designed for compound objects of unknown type. For builtins or objects of
+    known type, use PipeGraphPy.utils.functional.lazy.
+    """
+
+    def __init__(self, func):
+        """
+        Pass in a callable that returns the object to be wrapped.
+
+        If copies are made of the resulting SimpleLazyObject, which can happen
+        in various circumstances within PGP, then you must ensure that the
+        callable can be safely run more than once and will return the same
+        value.
+        """
+        self.__dict__['_setupfunc'] = func
+        super().__init__()
+
+    def _setup(self):
+        self._wrapped = self._setupfunc()
+
+    # Return a meaningful representation of the lazy object for debugging
+    # without evaluating the wrapped object.
+    def __repr__(self):
+        if self._wrapped is empty:
+            repr_attr = self._setupfunc
+        else:
+            repr_attr = self._wrapped
+        return '<%s: %r>' % (type(self).__name__, repr_attr)
+
+    def __copy__(self):
+        if self._wrapped is empty:
+            # If uninitialized, copy the wrapper. Use SimpleLazyObject, not
+            # self.__class__, because the latter is proxied.
+            return SimpleLazyObject(self._setupfunc)
+        else:
+            # If initialized, return a copy of the wrapped object.
+            return copy.copy(self._wrapped)
+
+    def __deepcopy__(self, memo):
+        if self._wrapped is empty:
+            # We have to use SimpleLazyObject, not self.__class__, because the
+            # latter is proxied.
+            result = SimpleLazyObject(self._setupfunc)
+            memo[id(self)] = result
+            return result
+        return copy.deepcopy(self._wrapped, memo)
+
+
+def partition(predicate, values):
+    """
+    Split the values into two sets, based on the return value of the function
+    (True/False). e.g.:
+
+        >>> partition(lambda x: x > 3, range(5))
+        [0, 1, 2, 3], [4]
+    """
+    results = ([], [])
+    for item in values:
+        results[predicate(item)].append(item)
+    return results

PipeGraphPy/utils/handle_graph.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import signal
+from PipeGraphPy.logger import log
+from PipeGraphPy.common import multi_graph
+
+
+def stop_running_with_exception(graph_id, err):
+    """停止运行"""
+    graph = multi_graph[graph_id]
+    graph.stop_run(err)
+
+
+class HandleGraph(object):
+    def __init__(self, id):
+        self.id = id
+        self.graph = multi_graph[self.id]
+
+    def __getattr__(self, name):
+        return getattr(self.graph, name)
+
+
+def kill_pid(pid):
+    """强制kill某个pid"""
+    try:
+        a = os.kill(pid, signal.SIGKILL)
+        # a = os.kill(pid, signal.9) #　与上等效
+        log.info("已杀死pid为%s的进程,　返回值是:%s" % (pid, a))
+    except Exception:
+        log.error("没有如此进程!!!")

PipeGraphPy/utils/lock.py

@@ -0,0 +1 @@
+# 连接redis

PipeGraphPy/utils/mq.py

@@ -0,0 +1,54 @@
+# #!/usr/bin/env python
+# # -*- coding: utf-8 -*-
+#
+# import traceback
+# from PipeGraphPy.logger import log
+# from PipeGraphPy.config import settings
+# from PipeGraphPy.db.models import PredictRecordTB, MqTB
+# from rabbitmqpy import Puber
+# from datetime import datetime, timedelta
+#
+#
+# def publish_predict_dict(predict_dict, **kwargs):
+#     # global puber
+#     puber = None
+#     try:
+#         if settings.RUN_PERMISSION:
+#             try:
+#                 if settings.DEBUG:
+#                     puber = Puber(
+#                         settings.AMQP_URL,
+#                         'PipeGraphPypub_predict_e_test',
+#                         'direct',
+#                         routing_key='PipeGraphPypub_predict_k_test'
+#                     )
+#                 else:
+#                     puber = Puber(
+#                         settings.AMQP_URL,
+#                         'PipeGraphPypub_predict_e',
+#                         'direct',
+#                         routing_key='PipeGraphPypub_predict_k'
+#                     )
+#             except:
+#                 puber = None
+#         else:
+#             puber = None
+#         if puber is not None:
+#             puber.send(predict_dict)
+#             log_info = '发送MQ：exchange=PipeGraphPypub_predict_e,routing_key=PipeGraphPypub_predict_k,body=%s' % (
+#                 str(predict_dict)[:100]
+#             )
+#             MqTB.add(
+#                 pubdate=int((datetime.utcnow()+timedelta(hours=8)).strftime("%Y%m%d")),
+#                 graph_id=predict_dict.get("id", 0),
+#                 exchange="PipeGraphPypub_predict_e",
+#                 queue="",
+#                 route_key="PipeGraphPypub_predict_k",
+#                 clock=predict_dict.get("clock", "12"),
+#                 kind = 1
+#                 )
+#             log.info(log_info, **kwargs)
+#             if kwargs.get('plog_record_id'):
+#                 PredictRecordTB.set(is_pub=1).where(id=kwargs['plog_record_id'])
+#     except Exception:
+#         log.error(traceback.format_exc(), **kwargs)

PipeGraphPy/utils/osutil.py

@@ -0,0 +1,29 @@
+import psutil
+import time
+
+def get_disk_mbps(interval=1):
+    disk_io_prev = psutil.disk_io_counters()
+    time.sleep(interval)
+    disk_io_now = psutil.disk_io_counters()
+    bytes_read = disk_io_now.read_bytes - disk_io_prev.read_bytes
+    bytes_written = disk_io_now.write_bytes - disk_io_prev.write_bytes
+    mbps_read = bytes_read / interval / 1024 / 1024
+    mbps_written = bytes_written / interval / 1024 / 1024
+    mbps = mbps_read + mbps_written
+    return int(mbps)
+
+
+def get_cpu_usage():
+    return psutil.cpu_percent()
+
+
+def get_memory_usage():
+    return psutil.virtual_memory().percent
+
+
+def get_disk_usage(path="/"):
+    return psutil.disk_usage(path).percent
+
+
+def get_load():
+    return psutil.getloadavg()