cavisson-pythonagent 0.0.1__py3-none-any.whl

pythonagent/vendor/pympler/garbagegraph.py

@@ -0,0 +1,80 @@
+
+from .refgraph import ReferenceGraph
+from .util.stringutils import trunc, pp
+
+import sys
+import gc
+
+__all__ = ['GarbageGraph', 'start_debug_garbage', 'end_debug_garbage']
+
+
+class GarbageGraph(ReferenceGraph):
+    """
+    The ``GarbageGraph`` is a ``ReferenceGraph`` that illustrates the objects
+    building reference cycles. The garbage collector is switched to debug mode
+    (all identified garbage is stored in `gc.garbage`) and the garbage
+    collector is invoked. The collected objects are then illustrated in a
+    directed graph.
+
+    Large graphs can be reduced to the actual cycles by passing ``reduce=True``
+    to the constructor.
+
+    It is recommended to disable the garbage collector when using the
+    ``GarbageGraph``.
+
+    >>> from pympler.garbagegraph import GarbageGraph, start_debug_garbage
+    >>> start_debug_garbage()
+    >>> l = []
+    >>> l.append(l)
+    >>> del l
+    >>> gb = GarbageGraph()
+    >>> gb.render('garbage.eps')
+    True
+    """
+    def __init__(self, reduce=False, collectable=True):
+        """
+        Initialize the GarbageGraph with the objects identified by the garbage
+        collector. If `collectable` is true, every reference cycle is recorded.
+        Otherwise only uncollectable objects are reported.
+        """
+        if collectable:
+            gc.set_debug(gc.DEBUG_SAVEALL)
+        else:
+            gc.set_debug(0)
+        gc.collect()
+
+        ReferenceGraph.__init__(self, gc.garbage, reduce)
+
+    def print_stats(self, stream=None):
+        """
+        Log annotated garbage objects to console or file.
+
+        :param stream: open file, uses sys.stdout if not given
+        """
+        if not stream:  # pragma: no cover
+            stream = sys.stdout
+        self.metadata.sort(key=lambda x: -x.size)
+        stream.write('%-10s %8s %-12s %-46s\n' % ('id', 'size', 'type',
+                                                  'representation'))
+        for g in self.metadata:
+            stream.write('0x%08x %8d %-12s %-46s\n' % (g.id, g.size,
+                                                       trunc(g.type, 12),
+                                                       trunc(g.str, 46)))
+        stream.write('Garbage: %8d collected objects (%s in cycles): %12s\n' %
+                     (self.count, self.num_in_cycles, pp(self.total_size)))
+
+
+def start_debug_garbage():
+    """
+    Turn off garbage collector to analyze *collectable* reference cycles.
+    """
+    gc.collect()
+    gc.disable()
+
+
+def end_debug_garbage():
+    """
+    Turn garbage collection on and disable debug output.
+    """
+    gc.set_debug(0)
+    gc.enable()

pythonagent/vendor/pympler/mprofile.py

@@ -0,0 +1,97 @@
+"""
+Memory usage profiler for Python.
+
+"""
+import inspect
+import sys
+
+from . import muppy
+
+
+class MProfiler(object):
+    """A memory usage profiler class.
+
+    Memory data for each function is stored as a 3-element list in the
+    dictionary self.memories. The index is always a codepoint (see below).
+    The following are the definitions of the members:
+
+    [0] = The number of times this function was called
+    [1] = Minimum memory consumption when this function was measured.
+    [2] = Maximum memory consumption when this function was measured.
+
+    A codepoint is a list of 3-tuple of the type
+    (filename, functionname, linenumber). You can omit either element, which
+    will cause the profiling to be triggered if any of the other criteria
+    match. E.g.
+    - (None, foo, None), will profile any foo function,
+    - (bar, foo, None) will profile only the foo function from the bar file,
+    - (bar, foo, 17) will profile only line 17 of the foo function defined
+      in the file bar.
+
+    Additionally, you can define on what events you want the profiling be
+    triggered. Possible events are defined in
+    http://docs.python.org/lib/debugger-hooks.html.
+
+    If you do not define either codepoints or events, the profiler will
+    record the memory usage in at every codepoint and event.
+
+    """
+
+    def __init__(self, codepoints=None, events=None):
+        """
+        keyword arguments:
+        codepoints -- a list of points in code to monitor (defaults to all
+            codepoints)
+        events -- a list of events to monitor (defaults to all events)
+        """
+        self.memories = {}
+        self.codepoints = codepoints
+        self.events = events
+
+    def codepoint_included(self, codepoint):
+        """Check if codepoint matches any of the defined codepoints."""
+        if self.codepoints is None:
+            return True
+        for cp in self.codepoints:
+            mismatch = False
+            for i in range(len(cp)):
+                if (cp[i] is not None) and (cp[i] != codepoint[i]):
+                    mismatch = True
+                    break
+            if not mismatch:
+                return True
+        return False
+
+    def profile(self, frame, event, arg):  # arg req to match signature
+        """Profiling method used to profile matching codepoints and events."""
+        if (self.events is None) or (event in self.events):
+            frame_info = inspect.getframeinfo(frame)
+            cp = (frame_info[0], frame_info[2], frame_info[1])
+            if self.codepoint_included(cp):
+                objects = muppy.get_objects()
+                size = muppy.get_size(objects)
+                if cp not in self.memories:
+                    self.memories[cp] = [0, 0, 0, 0]
+                    self.memories[cp][0] = 1
+                    self.memories[cp][1] = size
+                    self.memories[cp][2] = size
+                else:
+                    self.memories[cp][0] += 1
+                    if self.memories[cp][1] > size:
+                        self.memories[cp][1] = size
+                    if self.memories[cp][2] < size:
+                        self.memories[cp][2] = size
+
+    def run(self, cmd):
+        sys.setprofile(self.profile)
+        try:
+            exec(cmd)
+        finally:
+            sys.setprofile(None)
+        return self
+
+
+if __name__ == "__main__":
+    p = MProfiler()
+    p.run("print('hello')")
+    print(p.memories)

pythonagent/vendor/pympler/muppy.py

@@ -0,0 +1,275 @@
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple
+
+import gc
+
+from . import summary
+from .util import compat
+
+from inspect import isframe, stack
+
+from sys import getsizeof
+
+from .asizeof import _Py_TPFLAGS_HAVE_GC
+
+
+def ignore_object(obj: Any) -> bool:
+    try:
+        return isframe(obj)
+    except ReferenceError:
+        return True
+
+
+def get_objects(remove_dups: bool = True, include_frames: bool = False
+                ) -> List[Any]:
+    """Return a list of all known objects excluding frame objects.
+
+    If (outer) frame objects shall be included, pass `include_frames=True`.  In
+    order to prevent building reference cycles, the current frame object (of
+    the caller of get_objects) is ignored. This will not prevent creating
+    reference cycles if the object list is passed up the call-stack. Therefore,
+    frame objects are not included by default.
+
+    Keyword arguments:
+    remove_dups -- if True, all duplicate objects will be removed.
+    include_frames -- if True, includes frame objects.
+    """
+    gc.collect()
+
+    # Do not initialize local variables before calling gc.get_objects or those
+    # will be included in the list. Furthermore, ignore frame objects to
+    # prevent reference cycles.
+    tmp = gc.get_objects()
+    tmp = [o for o in tmp if not ignore_object(o)]
+
+    res = []
+    for o in tmp:
+        # gc.get_objects returns only container objects, but we also want
+        # the objects referenced by them
+        refs = get_referents(o)
+        for ref in refs:
+            if not gc.is_tracked(ref):
+                # we already got the container objects, now we only add
+                # non-container objects
+                res.append(ref)
+    res.extend(tmp)
+    if remove_dups:
+        res = _remove_duplicates(res)
+
+    if include_frames:
+        for sf in stack()[2:]:
+            res.append(sf[0])
+    return res
+
+
+def get_size(objects: List[Any]) -> int:
+    """Compute the total size of all elements in objects."""
+    res = 0
+    for o in objects:
+        try:
+            res += getsizeof(o)
+        except AttributeError:
+            print("IGNORING: type=%s; o=%s" % (str(type(o)), str(o)))
+    return res
+
+
+def get_diff(left: List[Any], right: List[Any]) -> Dict[str, List[Any]]:
+    """Get the difference of both lists.
+
+    The result will be a dict with this form {'+': [], '-': []}.
+    Items listed in '+' exist only in the right list,
+    items listed in '-' exist only in the left list.
+
+    """
+    res = {'+': [], '-': []}  # type: Dict[str, List[Any]]
+
+    def partition(objects: List[Any]) -> Dict[type, List[Any]]:
+        """Partition the passed object list."""
+        res = {}  # type: Dict[type, List[Any]]
+        for o in objects:
+            t = type(o)
+            if type(o) not in res:
+                res[t] = []
+            res[t].append(o)
+        return res
+
+    def get_not_included(foo: List[Any], bar: Dict[type, List[Any]]
+                         ) -> List[Any]:
+        """Compare objects from foo with objects defined in the values of
+        bar (set of partitions).
+        Returns a list of all objects included in list, but not dict values.
+        """
+        res = []  # type: List[Any]
+        for o in foo:
+            if not compat.object_in_list(type(o), bar):
+                res.append(o)
+            elif not compat.object_in_list(o, bar[type(o)]):
+                res.append(o)
+        return res
+
+    # Create partitions of both lists. This will reduce the time required for
+    # the comparison
+    left_objects = partition(left)
+    right_objects = partition(right)
+    # and then do the diff
+    res['+'] = get_not_included(right, left_objects)
+    res['-'] = get_not_included(left, right_objects)
+    return res
+
+
+def sort(objects: List[Any]) -> List[Any]:
+    """Sort objects by size in bytes."""
+    objects = sorted(objects, key=getsizeof)
+    return objects
+
+
+def filter(objects: List[Any], Type: Optional[type] = None, min: int = -1,
+           max: int = -1) -> List[Any]:
+    """Filter objects.
+
+    The filter can be by type, minimum size, and/or maximum size.
+
+    Keyword arguments:
+    Type -- object type to filter by
+    min -- minimum object size
+    max -- maximum object size
+
+    """
+    res = []  # type: List[Any]
+    if min > max and max > -1:
+        raise ValueError("minimum must be smaller than maximum")
+
+    if Type is not None:
+        objects = [o for o in objects if isinstance(o, Type)]
+    if min > -1:
+        objects = [o for o in objects if getsizeof(o) > min]
+    if max > -1:
+        objects = [o for o in objects if getsizeof(o) < max]
+    return objects
+
+
+def get_referents(object: Any, level: int = 1) -> List[Any]:
+    """Get all referents of an object up to a certain level.
+
+    The referents will not be returned in a specific order and
+    will not contain duplicate objects. Duplicate objects will be removed.
+
+    Keyword arguments:
+    level -- level of indirection to which referents considered.
+
+    This function is recursive.
+
+    """
+    res = gc.get_referents(object)
+    level -= 1
+    if level > 0:
+        for o in res:
+            res.extend(get_referents(o, level))
+    res = _remove_duplicates(res)
+    return res
+
+
+def _get_usage(function: Callable, *args: Any) -> Optional[List]:
+    """Test if more memory is used after the function has been called.
+
+    The function will be invoked twice and only the second measurement will be
+    considered. Thus, memory used in initialisation (e.g. loading modules)
+    will not be included in the result. The goal is to identify memory leaks
+    caused by functions which use more and more memory.
+
+    Any arguments next to the function will be passed on to the function
+    on invocation.
+
+    Note that this function is currently experimental, because it is not
+    tested thoroughly and performs poorly.
+
+    """
+    # The usage of a function is calculated by creating one summary of all
+    # objects before the function is invoked and afterwards. These summaries
+    # are compared and the diff is returned.
+    # This function works in a 2-steps process. Before the actual function is
+    # invoked an empty dummy function is measurement to identify the overhead
+    # involved in the measuring process. This overhead then is subtracted from
+    # the measurement performed on the passed function. The result reflects the
+    # actual usage of a function call.
+    # Also, a measurement is performed twice, allowing the adjustment to
+    # initializing things, e.g. modules
+
+    res = None
+
+    def _get_summaries(function: Callable, *args: Any) -> Tuple:
+        """Get a 2-tuple containing one summary from before, and one summary
+        from after the function has been invoked.
+
+        """
+        s_before = summary.summarize(get_objects())
+        function(*args)
+        s_after = summary.summarize(get_objects())
+        return (s_before, s_after)
+
+    def _get_usage(function: Callable, *args: Any) -> List:
+        """Get the usage of a function call.
+        This function is to be used only internally. The 'real' get_usage
+        function is a wrapper around _get_usage, but the workload is done
+        here.
+
+        """
+        # init before calling
+        (s_before, s_after) = _get_summaries(function, *args)
+        # ignore all objects used for the measurement
+        ignore = []
+        if s_before != s_after:
+            ignore.append(s_before)
+        for row in s_before:
+            # ignore refs from summary and frame (loop)
+            if len(gc.get_referrers(row)) == 2:
+                ignore.append(row)
+            for item in row:
+                # ignore refs from summary and frame (loop)
+                if len(gc.get_referrers(item)) == 2:
+                    ignore.append(item)
+        for o in ignore:
+            s_after = summary._subtract(s_after, o)
+        res = summary.get_diff(s_before, s_after)
+        return summary._sweep(res)
+
+    # calibrate; twice for initialization
+    def noop() -> None:
+        pass
+    offset = _get_usage(noop)
+    offset = _get_usage(noop)
+    # perform operation twice to handle objects possibly used in
+    # initialisation
+    tmp = _get_usage(function, *args)
+    tmp = _get_usage(function, *args)
+    tmp = summary.get_diff(offset, tmp)
+    tmp = summary._sweep(tmp)
+    if len(tmp) != 0:
+        res = tmp
+    return res
+
+
+def _is_containerobject(o: Any) -> bool:
+    """Is the passed object a container object."""
+    return bool(getattr(type(o), '__flags__', 0) & _Py_TPFLAGS_HAVE_GC)
+
+
+def _remove_duplicates(objects: List[Any]) -> List[Any]:
+    """Remove duplicate objects.
+
+    Inspired by http://www.peterbe.com/plog/uniqifiers-benchmark
+
+    """
+    seen = set()  # type: Set[int]
+    result = []
+    for item in objects:
+        marker = id(item)
+        if marker in seen:
+            continue
+        seen.add(marker)
+        result.append(item)
+    return result
+
+
+def print_summary() -> None:
+    """Print a summary of all known objects."""
+    summary.print_(summary.summarize(get_objects()))

pythonagent/vendor/pympler/panels.py

@@ -0,0 +1,115 @@
+"""
+Expose a memory-profiling panel to the Django Debug toolbar.
+
+Shows process memory information (virtual size, resident set size) and model
+instances for the current request.
+
+Requires Django and Django Debug toolbar:
+
+https://github.com/jazzband/django-debug-toolbar
+
+Pympler adds a memory panel as a third party addon (not included in the
+django-debug-toolbar). It can be added by overriding the `DEBUG_TOOLBAR_PANELS`
+setting in the Django project settings::
+
+    DEBUG_TOOLBAR_PANELS = (
+        'debug_toolbar.panels.timer.TimerDebugPanel',
+        'pympler.panels.MemoryPanel',
+        )
+
+Pympler also needs to be added to the `INSTALLED_APPS` in the Django settings::
+
+    INSTALLED_APPS = INSTALLED_APPS + ('debug_toolbar', 'pympler')
+"""
+
+from .classtracker import ClassTracker
+from .process import ProcessMemoryInfo
+from .util.stringutils import pp
+
+try:
+    from debug_toolbar.panels import Panel
+    from django.apps import apps
+    from django.template import Context, Template
+    from django.template.loader import render_to_string
+    from django.http.request import HttpRequest
+    from django.http.response import HttpResponse
+except ImportError:
+    class Panel(object):  # type: ignore
+        pass
+
+    class Template(object):  # type: ignore
+        pass
+
+    class Context(object):  # type: ignore
+        pass
+
+    class HttpRequest(object):  # type: ignore
+        pass
+
+    class HttpResponse(object):  # type: ignore
+        pass
+
+
+class MemoryPanel(Panel):
+
+    name = 'pympler'
+
+    title = 'Memory'
+
+    template = 'memory_panel.html'
+
+    classes = [Context, Template]
+
+    def process_request(self, request: HttpRequest) -> HttpResponse:
+        self._tracker = ClassTracker()
+        for cls in apps.get_models() + self.classes:
+            self._tracker.track_class(cls)
+        self._tracker.create_snapshot('before')
+        self.record_stats({'before': ProcessMemoryInfo()})
+        response = super(MemoryPanel, self).process_request(request)
+        self.record_stats({'after': ProcessMemoryInfo()})
+        self._tracker.create_snapshot('after')
+        stats = self._tracker.stats
+        stats.annotate()
+        self.record_stats({'stats': stats})
+        return response
+
+    def enable_instrumentation(self) -> None:
+        self._tracker = ClassTracker()
+        for cls in apps.get_models() + self.classes:
+            self._tracker.track_class(cls)
+
+    def disable_instrumentation(self) -> None:
+        self._tracker.detach_all_classes()
+
+    def nav_subtitle(self) -> str:
+        context = self.get_stats()
+        before = context['before']
+        after = context['after']
+        rss = after.rss
+        delta = rss - before.rss
+        delta = ('(+%s)' % pp(delta)) if delta > 0 else ''
+        return "%s %s" % (pp(rss), delta)
+
+    @property
+    def content(self) -> str:
+        context = self.get_stats()
+        before = context['before']
+        after = context['after']
+        stats = context['stats']
+        rows = [('Resident set size', after.rss),
+                ('Virtual size', after.vsz),
+                ]
+        rows.extend(after - before)
+        rows = [(key, pp(value)) for key, value in rows]
+        rows.extend(after.os_specific)
+
+        classes = []
+        snapshot = stats.snapshots[-1]
+        for model in stats.tracked_classes:
+            history = [cnt for _, cnt in stats.history[model]]
+            size = snapshot.classes.get(model, {}).get('sum', 0)
+            if history and history[-1] > 0:
+                classes.append((model, history, pp(size)))
+        context.update({'rows': rows, 'classes': classes})
+        return render_to_string(self.template, context)