cavisson-pythonagent 0.0.1__py3-none-any.whl

pythonagent/vendor/pympler/refgraph.py

@@ -0,0 +1,350 @@
+"""
+This module exposes utilities to illustrate objects and their references as
+(directed) graphs. The current implementation requires 'graphviz' to be
+installed.
+"""
+
+from .asizeof import Asizer, named_refs
+from .util.stringutils import safe_repr, trunc
+from gc import get_referents
+from subprocess import Popen, PIPE
+from copy import copy
+from sys import platform
+
+__all__ = ['ReferenceGraph']
+
+
+# Popen might lead to deadlocks when file descriptors are leaked to
+# sub-processes on Linux. On Windows, however, close_fds=True leads to
+# ValueError if stdin/stdout/stderr is piped:
+# http://code.google.com/p/pympler/issues/detail?id=28#c1
+popen_flags = {}
+if platform not in ['win32']:  # pragma: no branch
+    popen_flags['close_fds'] = True
+
+
+class _MetaObject(object):
+    """
+    The _MetaObject stores meta-information, like a string representation,
+    corresponding to each object passed to a ReferenceGraph.
+    """
+    __slots__ = ('size', 'id', 'type', 'str', 'group', 'cycle')
+
+    def __init__(self):
+        self.cycle = False
+
+
+class _Edge(object):
+    """
+    Describes a reference from one object `src` to another object `dst`.
+    """
+    __slots__ = ('src', 'dst', 'label', 'group')
+
+    def __init__(self, src, dst, label):
+        self.src = src
+        self.dst = dst
+        self.label = label
+        self.group = None
+
+    def __repr__(self):
+        return "<%08x => %08x, '%s', %s>" % (self.src, self.dst, self.label,
+                                             self.group)
+
+    def __hash__(self):
+        return (self.src, self.dst, self.label).__hash__()
+
+    def __eq__(self, other):
+        return self.__hash__() == other.__hash__()
+
+
+class ReferenceGraph(object):
+    """
+    The ReferenceGraph illustrates the references between a collection of
+    objects by rendering a directed graph. That requires that 'graphviz' is
+    installed.
+
+    >>> from pympler.refgraph import ReferenceGraph
+    >>> a = 42
+    >>> b = 'spam'
+    >>> c = {a: b}
+    >>> gb = ReferenceGraph([a,b,c])
+    >>> gb.render('spam.eps')
+    True
+    """
+    def __init__(self, objects, reduce=False):
+        """
+        Initialize the ReferenceGraph with a collection of `objects`.
+        """
+        self.objects = list(objects)
+        self.count = len(self.objects)
+        self.num_in_cycles = 'N/A'
+        self.edges = None
+
+        if reduce:
+            self.num_in_cycles = self._reduce_to_cycles()
+            self._reduced = self  # TODO: weakref?
+        else:
+            self._reduced = None
+
+        self._get_edges()
+        self._annotate_objects()
+
+    def _eliminate_leafs(self, graph):
+        """
+        Eliminate leaf objects - that are objects not referencing any other
+        objects in the list `graph`. Returns the list of objects without the
+        objects identified as leafs.
+        """
+        result = []
+        idset = set([id(x) for x in graph])
+        for n in graph:
+            refset = set([id(x) for x in get_referents(n)])
+            if refset.intersection(idset):
+                result.append(n)
+        return result
+
+    def _reduce_to_cycles(self):
+        """
+        Iteratively eliminate leafs to reduce the set of objects to only those
+        that build cycles. Return the number of objects involved in reference
+        cycles. If there are no cycles, `self.objects` will be an empty list
+        and this method returns 0.
+        """
+        cycles = self.objects[:]
+        cnt = 0
+        while cnt != len(cycles):
+            cnt = len(cycles)
+            cycles = self._eliminate_leafs(cycles)
+        self.objects = cycles
+        return len(self.objects)
+
+    def reduce_to_cycles(self):
+        """
+        Iteratively eliminate leafs to reduce the set of objects to only those
+        that build cycles. Return the reduced graph. If there are no cycles,
+        None is returned.
+        """
+        if not self._reduced:
+            reduced = copy(self)
+            reduced.objects = self.objects[:]
+            reduced.metadata = []
+            reduced.edges = []
+            self.num_in_cycles = reduced._reduce_to_cycles()
+            reduced.num_in_cycles = self.num_in_cycles
+            if self.num_in_cycles:
+                reduced._get_edges()
+                reduced._annotate_objects()
+                for meta in reduced.metadata:
+                    meta.cycle = True
+            else:
+                reduced = None
+            self._reduced = reduced
+        return self._reduced
+
+    def _get_edges(self):
+        """
+        Compute the edges for the reference graph.
+        The function returns a set of tuples (id(a), id(b), ref) if a
+        references b with the referent 'ref'.
+        """
+        idset = set([id(x) for x in self.objects])
+        self.edges = set([])
+        for n in self.objects:
+            refset = set([id(x) for x in get_referents(n)])
+            for ref in refset.intersection(idset):
+                label = ''
+                members = None
+                if isinstance(n, dict):
+                    members = n.items()
+                if not members:
+                    members = named_refs(n)
+                for (k, v) in members:
+                    if id(v) == ref:
+                        label = k
+                        break
+                self.edges.add(_Edge(id(n), ref, label))
+
+    def _annotate_groups(self):
+        """
+        Annotate the objects belonging to separate (non-connected) graphs with
+        individual indices.
+        """
+        g = {}
+        for x in self.metadata:
+            g[x.id] = x
+
+        idx = 0
+        for x in self.metadata:
+            if not hasattr(x, 'group'):
+                x.group = idx
+                idx += 1
+            neighbors = set()
+            for e in self.edges:
+                if e.src == x.id:
+                    neighbors.add(e.dst)
+                if e.dst == x.id:
+                    neighbors.add(e.src)
+            for nb in neighbors:
+                g[nb].group = min(x.group, getattr(g[nb], 'group', idx))
+
+        # Assign the edges to the respective groups. Both "ends" of the edge
+        # should share the same group so just use the first object's group.
+        for e in self.edges:
+            e.group = g[e.src].group
+
+        self._max_group = idx
+
+    def _filter_group(self, group):
+        """
+        Eliminate all objects but those which belong to `group`.
+        ``self.objects``, ``self.metadata`` and ``self.edges`` are modified.
+        Returns `True` if the group is non-empty. Otherwise returns `False`.
+        """
+        self.metadata = [x for x in self.metadata if x.group == group]
+        group_set = set([x.id for x in self.metadata])
+        self.objects = [obj for obj in self.objects if id(obj) in group_set]
+        self.count = len(self.metadata)
+        if self.metadata == []:
+            return False
+
+        self.edges = [e for e in self.edges if e.group == group]
+
+        del self._max_group
+
+        return True
+
+    def split(self):
+        """
+        Split the graph into sub-graphs. Only connected objects belong to the
+        same graph. `split` yields copies of the Graph object. Shallow copies
+        are used that only replicate the meta-information, but share the same
+        object list ``self.objects``.
+
+        >>> from pympler.refgraph import ReferenceGraph
+        >>> a = 42
+        >>> b = 'spam'
+        >>> c = {a: b}
+        >>> t = (1,2,3)
+        >>> rg = ReferenceGraph([a,b,c,t])
+        >>> for subgraph in rg.split():
+        ...   print (subgraph.index)
+        0
+        1
+        """
+        self._annotate_groups()
+        index = 0
+
+        for group in range(self._max_group):
+            subgraph = copy(self)
+            subgraph.metadata = self.metadata[:]
+            subgraph.edges = self.edges.copy()
+
+            if subgraph._filter_group(group):
+                subgraph.total_size = sum([x.size for x in subgraph.metadata])
+                subgraph.index = index
+                index += 1
+                yield subgraph
+
+    def split_and_sort(self):
+        """
+        Split the graphs into sub graphs and return a list of all graphs sorted
+        by the number of nodes. The graph with most nodes is returned first.
+        """
+        graphs = list(self.split())
+        graphs.sort(key=lambda x: -len(x.metadata))
+        for index, graph in enumerate(graphs):
+            graph.index = index
+        return graphs
+
+    def _annotate_objects(self):
+        """
+        Extract meta-data describing the stored objects.
+        """
+        self.metadata = []
+        sizer = Asizer()
+        sizes = sizer.asizesof(*self.objects)
+        self.total_size = sizer.total
+        for obj, sz in zip(self.objects, sizes):
+            md = _MetaObject()
+            md.size = sz
+            md.id = id(obj)
+            try:
+                md.type = obj.__class__.__name__
+            except (AttributeError, ReferenceError):  # pragma: no cover
+                md.type = type(obj).__name__
+            md.str = safe_repr(obj, clip=128)
+            self.metadata.append(md)
+
+    def _get_graphviz_data(self):
+        """
+        Emit a graph representing the connections between the objects described
+        within the metadata list. The text representation can be transformed to
+        a graph with graphviz. Returns a string.
+        """
+        s = []
+        header = '// Process this file with graphviz\n'
+        s.append(header)
+        s.append('digraph G {\n')
+        s.append('    node [shape=box];\n')
+        for md in self.metadata:
+            label = trunc(md.str, 48).replace('"', "'")
+            extra = ''
+            if md.type == 'instancemethod':
+                extra = ', color=red'
+            elif md.type == 'frame':
+                extra = ', color=orange'
+            s.append('    "X%s" [ label = "%s\\n%s" %s ];\n' %
+                     (hex(md.id)[1:], label, md.type, extra))
+        for e in self.edges:
+            extra = ''
+            if e.label == '__dict__':
+                extra = ',weight=100'
+            s.append('    X%s -> X%s [label="%s"%s];\n' %
+                     (hex(e.src)[1:], hex(e.dst)[1:], e.label, extra))
+
+        s.append('}\n')
+        return "".join(s)
+
+    def render(self, filename, cmd='dot', format='ps', unflatten=False):
+        """
+        Render the graph to `filename` using graphviz. The graphviz invocation
+        command may be overridden by specifying `cmd`. The `format` may be any
+        specifier recognized by the graph renderer ('-Txxx' command).  The
+        graph can be preprocessed by the *unflatten* tool if the `unflatten`
+        parameter is True.  If there are no objects to illustrate, the method
+        does not invoke graphviz and returns False. If the renderer returns
+        successfully (return code 0), True is returned.
+
+        An `OSError` is raised if the graphviz tool cannot be found.
+        """
+        if self.objects == []:
+            return False
+
+        data = self._get_graphviz_data()
+
+        options = ('-Nfontsize=10',
+                   '-Efontsize=10',
+                   '-Nstyle=filled',
+                   '-Nfillcolor=#E5EDB8',
+                   '-Ncolor=#CCCCCC')
+        cmdline = (cmd, '-T%s' % format, '-o', filename) + options
+
+        if unflatten:
+            p1 = Popen(('unflatten', '-l7'), stdin=PIPE, stdout=PIPE,
+                       **popen_flags)
+            p2 = Popen(cmdline, stdin=p1.stdout, **popen_flags)
+            p1.communicate(data.encode())
+            p2.communicate()
+            return p2.returncode == 0
+        else:
+            p = Popen(cmdline, stdin=PIPE, **popen_flags)
+            p.communicate(data.encode())
+            return p.returncode == 0
+
+    def write_graph(self, filename):
+        """
+        Write raw graph data which can be post-processed using graphviz.
+        """
+        f = open(filename, 'w')
+        f.write(self._get_graphviz_data())
+        f.close()

pythonagent/vendor/pympler/summary.py

@@ -0,0 +1,321 @@
+"""A collection of functions to summarize object information.
+
+This module provides several function which will help you to analyze object
+information which was gathered. Often it is sufficient to work with aggregated
+data instead of handling the entire set of existing objects. For example can a
+memory leak identified simple based on the number and size of existing objects.
+
+A summary contains information about objects in a table-like manner.
+Technically, it is a list of lists. Each of these lists represents a row,
+whereas the first column reflects the object type, the second column the number
+of objects, and the third column the size of all these objects. This allows a
+simple table-like output like the	following:
+
+=============  ============  =============
+       types     # objects     total size
+=============  ============  =============
+<type 'dict'>             2            560
+ <type 'str'>             3            126
+ <type 'int'>             4             96
+<type 'long'>             2             66
+<type 'list'>             1             40
+=============  ============  =============
+
+Another advantage of summaries is that they influence the system you analyze
+only to a minimum. Working with references to existing objects will keep these
+objects alive. Most of the times this is no desired behavior (as it will have
+an impact on the observations). Using summaries reduces this effect greatly.
+
+output representation
+---------------------
+
+The output representation of types is defined in summary.representations.
+Every type defined in this dictionary will be represented as specified. Each
+definition has a list of different representations. The later a representation
+appears in this list, the higher its verbosity level. From types which are not
+defined in summary.representations the default str() representation will be
+used.
+
+Per default, summaries will use the verbosity level 1 for any encountered type.
+The reason is that several computations are done with summaries and rows have
+to remain comparable. Therefore information which reflect an objects state,
+e.g. the current line number of a frame, should not be included. You may add
+more detailed information at higher verbosity levels than 1.
+"""
+
+import re
+import sys
+import types
+
+from .util import stringutils
+from sys import getsizeof
+
+representations = {}
+
+
+def _init_representations():
+    global representations
+    if sys.hexversion < 0x2040000:
+        classobj = [
+            lambda c: "classobj(%s)" % repr(c),
+        ]
+        representations[types.ClassType] = classobj
+        instance = [
+            lambda f: "instance(%s)" % repr(f.__class__),
+        ]
+        representations[types.InstanceType] = instance
+        instancemethod = [
+            lambda i: "instancemethod (%s)" % (repr(i.im_func)),
+            lambda i: "instancemethod (%s, %s)" % (repr(i.im_class),
+                                                   repr(i.im_func)),
+        ]
+        representations[types.MethodType] = instancemethod
+    frame = [
+        lambda f: "frame (codename: %s)" % (f.f_code.co_name),
+        lambda f: "frame (codename: %s, codeline: %s)" %
+                  (f.f_code.co_name, f.f_code.co_firstlineno),
+        lambda f: "frame (codename: %s, filename: %s, codeline: %s)" %
+                  (f.f_code.co_name, f.f_code.co_filename,
+                   f.f_code.co_firstlineno)
+    ]
+    representations[types.FrameType] = frame
+    _dict = [
+        lambda d: str(type(d)),
+        lambda d: "dict, len=%s" % len(d),
+    ]
+    representations[dict] = _dict
+    function = [
+        lambda f: "function (%s)" % f.__name__,
+        lambda f: "function (%s.%s)" % (f.__module__, f.__name__),
+    ]
+    representations[types.FunctionType] = function
+    _list = [
+        lambda l: str(type(l)),
+        lambda l: "list, len=%s" % len(l)
+    ]
+    representations[list] = _list
+    module = [lambda m: "module(%s)" % getattr(
+        m, '__name__', getattr(m, '__file__', 'nameless, id: %d' % id(m))
+    )]
+    representations[types.ModuleType] = module
+    _set = [
+        lambda s: str(type(s)),
+        lambda s: "set, len=%s" % len(s)
+    ]
+    representations[set] = _set
+
+
+_init_representations()
+
+
+def summarize(objects):
+    """Summarize an objects list.
+
+    Return a list of lists, whereas each row consists of::
+      [str(type), number of objects of this type, total size of these objects].
+
+    No guarantee regarding the order is given.
+
+    """
+    count = {}
+    total_size = {}
+    for o in objects:
+        otype = _repr(o)
+        if otype in count:
+            count[otype] += 1
+            total_size[otype] += getsizeof(o)
+        else:
+            count[otype] = 1
+            total_size[otype] = getsizeof(o)
+    rows = []
+    for otype in count:
+        rows.append([otype, count[otype], total_size[otype]])
+    return rows
+
+
+def get_diff(left, right):
+    """Get the difference of two summaries.
+
+    Subtracts the values of the right summary from the values of the left
+    summary.
+    If similar rows appear on both sides, the are included in the summary with
+    0 for number of elements and total size.
+    If the number of elements of a row of the diff is 0, but the total size is
+    not, it means that objects likely have changed, but not there number, thus
+    resulting in a changed size.
+
+    """
+    res = []
+
+    right_by_key = dict((r[0], r) for r in right)
+    left_by_key = dict((r[0], r) for r in left)
+
+    keys = set(right_by_key)
+    keys.update(left_by_key)
+
+    for key in keys:
+        r = right_by_key.get(key)
+        l = left_by_key.get(key)
+        if l and r:
+            res.append([key, r[1] - l[1], r[2] - l[2]])
+        elif r:
+            res.append(r)
+        elif l:
+            res.append([key, -l[1], -l[2]])
+        else:
+            continue  # shouldn't happen
+    return res
+
+
+def format_(rows, limit=15, sort='size', order='descending'):
+    """Format the rows as a summary.
+
+    Keyword arguments:
+    limit -- the maximum number of elements to be listed
+    sort  -- sort elements by 'size', 'type', or '#'
+    order -- sort 'ascending' or 'descending'
+    """
+    localrows = []
+    for row in rows:
+        localrows.append(list(row))
+    # input validation
+    sortby = ['type', '#', 'size']
+    if sort not in sortby:
+        raise ValueError("invalid sort, should be one of" + str(sortby))
+    orders = ['ascending', 'descending']
+    if order not in orders:
+        raise ValueError("invalid order, should be one of" + str(orders))
+    # sort rows
+    if sortby.index(sort) == 0:
+        if order == "ascending":
+            localrows.sort(key=lambda x: _repr(x[0]))
+        elif order == "descending":
+            localrows.sort(key=lambda x: _repr(x[0]), reverse=True)
+    else:
+        if order == "ascending":
+            localrows.sort(key=lambda x: x[sortby.index(sort)])
+        elif order == "descending":
+            localrows.sort(key=lambda x: x[sortby.index(sort)], reverse=True)
+    # limit rows
+    localrows = localrows[0:limit]
+    for row in localrows:
+        row[2] = stringutils.pp(row[2])
+    # print rows
+    localrows.insert(0, ["types", "# objects", "total size"])
+    return _format_table(localrows)
+
+
+def _format_table(rows, header=True):
+    """Format a list of lists as a pretty table.
+
+    Keyword arguments:
+    header -- if True the first row is treated as a table header
+
+    inspired by http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/267662
+    """
+    border = "="
+    # vertical delimiter
+    vdelim = " | "
+    # padding nr. of spaces are left around the longest element in the
+    # column
+    padding = 1
+    # may be left,center,right
+    justify = 'right'
+    justify = {'left': str.ljust,
+               'center': str.center,
+               'right': str.rjust}[justify.lower()]
+    # calculate column widths (longest item in each col
+    # plus "padding" nr of spaces on both sides)
+    cols = zip(*rows)
+    colWidths = [max([len(str(item)) + 2 * padding for item in col])
+                 for col in cols]
+    borderline = vdelim.join([w * border for w in colWidths])
+    for row in rows:
+        yield vdelim.join([justify(str(item), width)
+                           for (item, width) in zip(row, colWidths)])
+        if header:
+            yield borderline
+            header = False
+
+
+def print_(rows, limit=15, sort='size', order='descending'):
+    """Print the rows as a summary.
+
+    Keyword arguments:
+    limit -- the maximum number of elements to be listed
+    sort  -- sort elements by 'size', 'type', or '#'
+    order -- sort 'ascending' or 'descending'
+
+    """
+    for line in format_(rows, limit=limit, sort=sort, order=order):
+        print(line)
+
+
+# regular expressions used by _repr to replace default type representations
+type_repr = re.compile(r"^<(type|class) '(\S+)'>$")
+address = re.compile(r' at 0x[0-9a-f]+')
+
+
+def _repr(o, verbosity=1):
+    """Get meaning object representation.
+
+    This function should be used when the simple str(o) output would result in
+    too general data. E.g. "<type 'instance'" is less meaningful than
+    "instance: Foo".
+
+    Keyword arguments:
+    verbosity -- if True the first row is treated as a table header
+
+    """
+    res = ""
+
+    t = type(o)
+    if (verbosity == 0) or (t not in representations):
+        res = str(t)
+    else:
+        verbosity -= 1
+        if len(representations[t]) <= verbosity:
+            verbosity = len(representations[t]) - 1
+        res = representations[t][verbosity](o)
+
+    res = address.sub('', res)
+    res = type_repr.sub(r'\2', res)
+
+    return res
+
+
+def _traverse(summary, function, *args):
+    """Traverse all objects of a summary and call function with each as a
+    parameter.
+
+    Using this function, the following objects will be traversed:
+    - the summary
+    - each row
+    - each item of a row
+    """
+    function(summary, *args)
+    for row in summary:
+        function(row, *args)
+        for item in row:
+            function(item, *args)
+
+
+def _subtract(summary, o):
+    """Remove object o from the summary by subtracting it's size."""
+    found = False
+    row = [_repr(o), 1, getsizeof(o)]
+    for r in summary:
+        if r[0] == row[0]:
+            (r[1], r[2]) = (r[1] - row[1], r[2] - row[2])
+            found = True
+    if not found:
+        summary.append([row[0], -row[1], -row[2]])
+    return summary
+
+
+def _sweep(summary):
+    """Remove all rows in which the total size and the total number of
+    objects is zero.
+
+    """
+    return [row for row in summary if ((row[2] != 0) or (row[1] != 0))]