cavisson-pythonagent 0.0.1__py3-none-any.whl

pythonagent/vendor/pympler/tracker.py

@@ -0,0 +1,267 @@
+"""The tracker module allows you to track changes in the memory usage over
+time.
+
+Using the SummaryTracker, you can create summaries and compare them
+with each other. Stored summaries can be ignored during comparison,
+avoiding the observer effect.
+
+The ObjectTracker allows to monitor object creation. You create objects from
+one time and compare with objects from an earlier time.
+
+"""
+import gc
+import inspect
+
+from . import muppy, summary
+from .util import compat
+
+
+class SummaryTracker(object):
+    """ Helper class to track changes between two summaries taken.
+
+    Detailed information on single objects will be lost, e.g. object size or
+    object id. But often summaries are sufficient to monitor the memory usage
+    over the lifetime of an application.
+
+    On initialisation, a first summary is taken. Every time `diff` is called,
+    a new summary will be created. Thus, a diff between the new and the last
+    summary can be extracted.
+
+    Be aware that filtering out previous summaries is time-intensive. You
+    should therefore restrict yourself to the number of summaries you really
+    need.
+
+    """
+    def __init__(self, ignore_self=True):
+        """Constructor.
+
+        The number of summaries managed by the tracker has a performance
+        impact on new summaries, iff you decide to exclude them from further
+        summaries. Therefore it is suggested to use them economically.
+
+        Keyword arguments:
+        ignore_self -- summaries managed by this object will be ignored.
+        """
+        self.s0 = summary.summarize(muppy.get_objects())
+        self.summaries = {}
+        self.ignore_self = ignore_self
+
+    def create_summary(self):
+        """Return a summary.
+
+        See also the notes on ignore_self in the class as well as the
+        initializer documentation.
+
+        """
+        if not self.ignore_self:
+            res = summary.summarize(muppy.get_objects())
+        else:
+            # If the user requested the data required to store summaries to be
+            # ignored in the summaries, we need to identify all objects which
+            # are related to each summary stored.
+            # Thus we build a list of all objects used for summary storage as
+            # well as a dictionary which tells us how often an object is
+            # referenced by the summaries.
+            # During this identification process, more objects are referenced,
+            # namely int objects identifying referenced objects as well as the
+            # corresponding count.
+            # For all these objects it will be checked whether they are
+            # referenced from outside the monitor's scope. If not, they will be
+            # subtracted from the snapshot summary, otherwise they are
+            # included (as this indicates that they are relevant to the
+            # application).
+
+            all_of_them = []  # every single object
+            ref_counter = {}  # how often it is referenced; (id(o), o) pairs
+
+            def store_info(o):
+                all_of_them.append(o)
+                if id(o) in ref_counter:
+                    ref_counter[id(o)] += 1
+                else:
+                    ref_counter[id(o)] = 1
+
+            # store infos on every single object related to the summaries
+            store_info(self.summaries)
+            for k, v in self.summaries.items():
+                store_info(k)
+                summary._traverse(v, store_info)
+
+            # do the summary
+            res = summary.summarize(muppy.get_objects())
+
+            # remove ids stored in the ref_counter
+            for _id in ref_counter:
+                # referenced in frame, ref_counter, ref_counter.keys()
+                if len(gc.get_referrers(_id)) == (3):
+                    summary._subtract(res, _id)
+            for o in all_of_them:
+                # referenced in frame, summary, all_of_them
+                if len(gc.get_referrers(o)) == (ref_counter[id(o)] + 2):
+                    summary._subtract(res, o)
+
+        return res
+
+    def diff(self, summary1=None, summary2=None):
+        """Compute diff between to summaries.
+
+        If no summary is provided, the diff from the last to the current
+        summary is used. If summary1 is provided the diff from summary1
+        to the current summary is used. If summary1 and summary2 are
+        provided, the diff between these two is used.
+
+        """
+        res = None
+        if summary2 is None:
+            self.s1 = self.create_summary()
+            if summary1 is None:
+                res = summary.get_diff(self.s0, self.s1)
+            else:
+                res = summary.get_diff(summary1, self.s1)
+            self.s0 = self.s1
+        else:
+            if summary1 is not None:
+                res = summary.get_diff(summary1, summary2)
+            else:
+                raise ValueError(
+                    "You cannot provide summary2 without summary1.")
+        return summary._sweep(res)
+
+    def print_diff(self, summary1=None, summary2=None):
+        """Compute diff between to summaries and print it.
+
+        If no summary is provided, the diff from the last to the current
+        summary is used. If summary1 is provided the diff from summary1
+        to the current summary is used. If summary1 and summary2 are
+        provided, the diff between these two is used.
+        """
+        summary.print_(self.diff(summary1=summary1, summary2=summary2))
+
+    def format_diff(self, summary1=None, summary2=None):
+        """Compute diff between to summaries and return a list of formatted
+        lines.
+
+        If no summary is provided, the diff from the last to the current
+        summary is used. If summary1 is provided the diff from summary1
+        to the current summary is used. If summary1 and summary2 are
+        provided, the diff between these two is used.
+        """
+        return summary.format_(self.diff(summary1=summary1, summary2=summary2))
+
+    def store_summary(self, key):
+        """Store a current summary in self.summaries."""
+        self.summaries[key] = self.create_summary()
+
+
+class ObjectTracker(object):
+    """
+    Helper class to track changes in the set of existing objects.
+
+    Each time you invoke a diff with this tracker, the objects which existed
+    during the last invocation are compared with the objects which exist during
+    the current invocation.
+
+    Please note that in order to do so, strong references to all objects will
+    be stored. This means that none of these objects can be garbage collected.
+    A use case for the ObjectTracker is the monitoring of a state which should
+    be stable, but you see new objects being created nevertheless. With the
+    ObjectTracker you can identify these new objects.
+
+    """
+
+    # Some precaution needs to be taken when handling frame objects (see
+    # warning at http://docs.python.org/lib/inspect-stack.html). All ignore
+    # lists used need to be emptied so no frame objects remain referenced.
+
+    def __init__(self):
+        """On initialisation, the current state of objects is stored.
+
+        Note that all objects which exist at this point in time will not be
+        released until you destroy this ObjectTracker instance.
+        """
+        self.o0 = self._get_objects(ignore=(inspect.currentframe(),))
+
+    def _get_objects(self, ignore=()):
+        """Get all currently existing objects.
+
+        XXX - ToDo: This method is a copy&paste from muppy.get_objects, but
+        some modifications are applied. Specifically, it allows to ignore
+        objects (which includes the current frame).
+
+        keyword arguments
+        ignore -- list of objects to ignore
+        """
+        def remove_ignore(objects, ignore=()):
+            # remove all objects listed in the ignore list
+            res = []
+            for o in objects:
+                if not compat.object_in_list(o, ignore):
+                    res.append(o)
+            return res
+
+        tmp = gc.get_objects()
+        ignore += (inspect.currentframe(), self, ignore, remove_ignore)
+        if hasattr(self, 'o0'):
+            ignore += (self.o0,)
+        if hasattr(self, 'o1'):
+            ignore += (self.o1,)
+        # this implies that referenced objects are also ignored
+        tmp = remove_ignore(tmp, ignore)
+        res = []
+        for o in tmp:
+            # gc.get_objects returns only container objects, but we also want
+            # the objects referenced by them
+            refs = muppy.get_referents(o)
+            for ref in refs:
+                if not gc.is_tracked(ref):
+                    # we already got the container objects, now we only add
+                    # non-container objects
+                    res.append(ref)
+        res.extend(tmp)
+        res = muppy._remove_duplicates(res)
+        if ignore is not None:
+            # repeat to filter out objects which may have been referenced
+            res = remove_ignore(res, ignore)
+        # manual cleanup, see comment above
+        del ignore
+        return res
+
+    def get_diff(self, ignore=()):
+        """Get the diff to the last time the  state of objects was measured.
+
+        keyword arguments
+        ignore -- list of objects to ignore
+        """
+        # ignore this and the caller frame
+        self.o1 = self._get_objects(ignore+(inspect.currentframe(),))
+        diff = muppy.get_diff(self.o0, self.o1)
+        self.o0 = self.o1
+        # manual cleanup, see comment above
+        return diff
+
+    def print_diff(self, ignore=()):
+        """Print the diff to the last time the state of objects was measured.
+
+        keyword arguments
+        ignore -- list of objects to ignore
+        """
+        # ignore this and the caller frame
+        for line in self.format_diff(ignore+(inspect.currentframe(),)):
+            print(line)
+
+    def format_diff(self, ignore=()):
+        """Format the diff to the last time the state of objects was measured.
+
+        keyword arguments
+        ignore -- list of objects to ignore
+        """
+        # ignore this and the caller frame
+        lines = []
+        diff = self.get_diff(ignore+(inspect.currentframe(),))
+        lines.append("Added objects:")
+        for line in summary.format_(summary.summarize(diff['+'])):
+            lines.append(line)
+        lines.append("Removed objects:")
+        for line in summary.format_(summary.summarize(diff['-'])):
+            lines.append(line)
+        return lines