jaseci 1.4.0.18__py3-none-any.whl → 1.4.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of jaseci might be problematic. Click here for more details.

Files changed (42) hide show
  1. jaseci/VERSION +1 -1
  2. jaseci/cli_tools/jsctl.py +41 -2
  3. jaseci/cli_tools/tests/test_jsctl.py +11 -0
  4. jaseci/extens/act_lib/internal.py +2 -1
  5. jaseci/extens/act_lib/std.py +7 -0
  6. jaseci/extens/act_lib/tests/test_std_lib.py +3 -1
  7. jaseci/extens/api/jsorc_api.py +41 -1
  8. jaseci/extens/api/walker_api.py +9 -0
  9. jaseci/jac/interpreter/architype_interp.py +63 -28
  10. jaseci/jac/interpreter/interp.py +54 -158
  11. jaseci/jac/interpreter/sentinel_interp.py +73 -5
  12. jaseci/jac/interpreter/walker_interp.py +27 -38
  13. jaseci/jac/ir/ast.py +9 -1
  14. jaseci/jac/jac.g4 +5 -4
  15. jaseci/jac/jac_parse/jacListener.py +8 -8
  16. jaseci/jac/jac_parse/jacParser.py +1167 -1154
  17. jaseci/jac/machine/jac_scope.py +23 -30
  18. jaseci/jac/machine/machine_state.py +76 -12
  19. jaseci/jsorc/jsorc.py +92 -79
  20. jaseci/jsorc/live_actions.py +29 -24
  21. jaseci/jsorc/redis.py +7 -5
  22. jaseci/prim/{action.py → ability.py} +44 -31
  23. jaseci/prim/architype.py +26 -8
  24. jaseci/prim/obj_mixins.py +5 -0
  25. jaseci/prim/sentinel.py +3 -1
  26. jaseci/prim/walker.py +7 -5
  27. jaseci/tests/jac_test_progs.py +9 -0
  28. jaseci/tests/test_jac.py +3 -3
  29. jaseci/tests/test_node.py +9 -12
  30. jaseci/tests/test_progs.py +16 -1
  31. jaseci/tests/test_stack.py +22 -0
  32. jaseci/utils/actions/actions_optimizer.py +23 -8
  33. jaseci/utils/gprof2dot.py +3786 -0
  34. jaseci/utils/json_handler.py +5 -1
  35. jaseci/utils/utils.py +52 -21
  36. {jaseci-1.4.0.18.dist-info → jaseci-1.4.0.20.dist-info}/METADATA +2 -2
  37. {jaseci-1.4.0.18.dist-info → jaseci-1.4.0.20.dist-info}/RECORD +41 -41
  38. jaseci/prim/item.py +0 -29
  39. {jaseci-1.4.0.18.dist-info → jaseci-1.4.0.20.dist-info}/LICENSE +0 -0
  40. {jaseci-1.4.0.18.dist-info → jaseci-1.4.0.20.dist-info}/WHEEL +0 -0
  41. {jaseci-1.4.0.18.dist-info → jaseci-1.4.0.20.dist-info}/entry_points.txt +0 -0
  42. {jaseci-1.4.0.18.dist-info → jaseci-1.4.0.20.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,3786 @@
1
+ #!/usr/bin/env python3
2
+ #
3
+ # Copyright 2008-2017 Jose Fonseca
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify it
6
+ # under the terms of the GNU Lesser General Public License as published
7
+ # by the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU Lesser General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU Lesser General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ #
18
+
19
+ """Generate a dot graph from the output of several profilers."""
20
+
21
+ __author__ = "Jose Fonseca et al"
22
+
23
+
24
+ import sys
25
+ import math
26
+ import os.path
27
+ import re
28
+ import textwrap
29
+ import optparse
30
+ import xml.parsers.expat
31
+ import collections
32
+ import locale
33
+ import json
34
+ import fnmatch
35
+
36
+ # Python 2.x/3.x compatibility
37
+ if sys.version_info[0] >= 3:
38
+ PYTHON_3 = True
39
+
40
+ def compat_iteritems(x):
41
+ return x.items() # No iteritems() in Python 3
42
+
43
+ def compat_itervalues(x):
44
+ return x.values() # No itervalues() in Python 3
45
+
46
+ def compat_keys(x):
47
+ return list(x.keys()) # keys() is a generator in Python 3
48
+
49
+ basestring = str # No class basestring in Python 3
50
+ unichr = chr # No unichr in Python 3
51
+ xrange = range # No xrange in Python 3
52
+ else:
53
+ PYTHON_3 = False
54
+
55
+ def compat_iteritems(x):
56
+ return x.iteritems()
57
+
58
+ def compat_itervalues(x):
59
+ return x.itervalues()
60
+
61
+ def compat_keys(x):
62
+ return x.keys()
63
+
64
+
65
+ ########################################################################
66
+ # Model
67
+
68
+
69
+ MULTIPLICATION_SIGN = unichr(0xD7)
70
+
71
+
72
+ def times(x):
73
+ return "%u%s" % (x, MULTIPLICATION_SIGN)
74
+
75
+
76
+ def percentage(p):
77
+ return "%.02f%%" % (p * 100.0,)
78
+
79
+
80
+ def add(a, b):
81
+ return a + b
82
+
83
+
84
+ def fail(a, b):
85
+ assert False
86
+
87
+
88
+ tol = 2**-23
89
+
90
+
91
+ def ratio(numerator, denominator):
92
+ try:
93
+ ratio = float(numerator) / float(denominator)
94
+ except ZeroDivisionError:
95
+ # 0/0 is undefined, but 1.0 yields more useful results
96
+ return 1.0
97
+ if ratio < 0.0:
98
+ if ratio < -tol:
99
+ sys.stderr.write(
100
+ "warning: negative ratio (%s/%s)\n" % (numerator, denominator)
101
+ )
102
+ return 0.0
103
+ if ratio > 1.0:
104
+ if ratio > 1.0 + tol:
105
+ sys.stderr.write(
106
+ "warning: ratio greater than one (%s/%s)\n" % (numerator, denominator)
107
+ )
108
+ return 1.0
109
+ return ratio
110
+
111
+
112
+ class UndefinedEvent(Exception):
113
+ """Raised when attempting to get an event which is undefined."""
114
+
115
+ def __init__(self, event):
116
+ Exception.__init__(self)
117
+ self.event = event
118
+
119
+ def __str__(self):
120
+ return "unspecified event %s" % self.event.name
121
+
122
+
123
+ class Event(object):
124
+ """Describe a kind of event, and its basic operations."""
125
+
126
+ def __init__(self, name, null, aggregator, formatter=str):
127
+ self.name = name
128
+ self._null = null
129
+ self._aggregator = aggregator
130
+ self._formatter = formatter
131
+
132
+ def __eq__(self, other):
133
+ return self is other
134
+
135
+ def __hash__(self):
136
+ return id(self)
137
+
138
+ def null(self):
139
+ return self._null
140
+
141
+ def aggregate(self, val1, val2):
142
+ """Aggregate two event values."""
143
+ assert val1 is not None
144
+ assert val2 is not None
145
+ return self._aggregator(val1, val2)
146
+
147
+ def format(self, val):
148
+ """Format an event value."""
149
+ assert val is not None
150
+ return self._formatter(val)
151
+
152
+
153
+ CALLS = Event("Calls", 0, add, times)
154
+ SAMPLES = Event("Samples", 0, add, times)
155
+ SAMPLES2 = Event("Samples", 0, add, times)
156
+
157
+ # Count of samples where a given function was either executing or on the stack.
158
+ # This is used to calculate the total time ratio according to the
159
+ # straightforward method described in Mike Dunlavey's answer to
160
+ # stackoverflow.com/questions/1777556/alternatives-to-gprof, item 4 (the myth
161
+ # "that recursion is a tricky confusing issue"), last edited 2012-08-30: it's
162
+ # just the ratio of TOTAL_SAMPLES over the number of samples in the profile.
163
+ #
164
+ # Used only when totalMethod == callstacks
165
+ TOTAL_SAMPLES = Event("Samples", 0, add, times)
166
+
167
+ TIME = Event("Time", 0.0, add, lambda x: "(" + str(x) + ")")
168
+ TIME_RATIO = Event("Time ratio", 0.0, add, lambda x: "(" + percentage(x) + ")")
169
+ TOTAL_TIME = Event("Total time", 0.0, fail)
170
+ TOTAL_TIME_RATIO = Event("Total time ratio", 0.0, fail, percentage)
171
+
172
+ labels = {
173
+ "self-time": TIME,
174
+ "self-time-percentage": TIME_RATIO,
175
+ "total-time": TOTAL_TIME,
176
+ "total-time-percentage": TOTAL_TIME_RATIO,
177
+ }
178
+ defaultLabelNames = ["total-time-percentage", "self-time-percentage"]
179
+
180
+ totalMethod = "callratios"
181
+
182
+
183
+ class Object(object):
184
+ """Base class for all objects in profile which can store events."""
185
+
186
+ def __init__(self, events=None):
187
+ if events is None:
188
+ self.events = {}
189
+ else:
190
+ self.events = events
191
+
192
+ def __hash__(self):
193
+ return id(self)
194
+
195
+ def __eq__(self, other):
196
+ return self is other
197
+
198
+ def __lt__(self, other):
199
+ return id(self) < id(other)
200
+
201
+ def __contains__(self, event):
202
+ return event in self.events
203
+
204
+ def __getitem__(self, event):
205
+ try:
206
+ return self.events[event]
207
+ except KeyError:
208
+ raise UndefinedEvent(event)
209
+
210
+ def __setitem__(self, event, value):
211
+ if value is None:
212
+ if event in self.events:
213
+ del self.events[event]
214
+ else:
215
+ self.events[event] = value
216
+
217
+
218
+ class Call(Object):
219
+ """A call between functions.
220
+
221
+ There should be at most one call object for every pair of functions.
222
+ """
223
+
224
+ def __init__(self, callee_id):
225
+ Object.__init__(self)
226
+ self.callee_id = callee_id
227
+ self.ratio = None
228
+ self.weight = None
229
+
230
+
231
+ class Function(Object):
232
+ """A function."""
233
+
234
+ def __init__(self, id, name):
235
+ Object.__init__(self)
236
+ self.id = id
237
+ self.name = name
238
+ self.module = None
239
+ self.process = None
240
+ self.calls = {}
241
+ self.called = None
242
+ self.weight = None
243
+ self.cycle = None
244
+ self.filename = None
245
+
246
+ def add_call(self, call):
247
+ if call.callee_id in self.calls:
248
+ sys.stderr.write(
249
+ "warning: overwriting call from function %s to %s\n"
250
+ % (str(self.id), str(call.callee_id))
251
+ )
252
+ self.calls[call.callee_id] = call
253
+
254
+ def get_call(self, callee_id):
255
+ if not callee_id in self.calls:
256
+ call = Call(callee_id)
257
+ call[SAMPLES] = 0
258
+ call[SAMPLES2] = 0
259
+ call[CALLS] = 0
260
+ self.calls[callee_id] = call
261
+ return self.calls[callee_id]
262
+
263
+ _parenthesis_re = re.compile(r"\([^()]*\)")
264
+ _angles_re = re.compile(r"<[^<>]*>")
265
+ _const_re = re.compile(r"\s+const$")
266
+
267
+ def stripped_name(self):
268
+ """Remove extraneous information from C++ demangled function names."""
269
+
270
+ name = self.name
271
+
272
+ # Strip function parameters from name by recursively removing paired parenthesis
273
+ while True:
274
+ name, n = self._parenthesis_re.subn("", name)
275
+ if not n:
276
+ break
277
+
278
+ # Strip const qualifier
279
+ name = self._const_re.sub("", name)
280
+
281
+ # Strip template parameters from name by recursively removing paired angles
282
+ while True:
283
+ name, n = self._angles_re.subn("", name)
284
+ if not n:
285
+ break
286
+
287
+ return name
288
+
289
+ # TODO: write utility functions
290
+
291
+ def __repr__(self):
292
+ return self.name
293
+
294
+ def dump(self, sep1=",\n\t", sep2=":=", sep3="\n"):
295
+ """Returns as a string all information available in this Function object
296
+ separators sep1:between entries
297
+ sep2:between attribute name and value,
298
+ sep3: inserted at end
299
+ """
300
+ return (
301
+ sep1.join("".join(k, sep2, v) for (k, v) in sorted(self.__dict__.items()))
302
+ + sep3
303
+ )
304
+
305
+
306
+ class Cycle(Object):
307
+ """A cycle made from recursive function calls."""
308
+
309
+ def __init__(self):
310
+ Object.__init__(self)
311
+ self.functions = set()
312
+
313
+ def add_function(self, function):
314
+ assert function not in self.functions
315
+ self.functions.add(function)
316
+ if function.cycle is not None:
317
+ for other in function.cycle.functions:
318
+ if function not in self.functions:
319
+ self.add_function(other)
320
+ function.cycle = self
321
+
322
+
323
+ class Profile(Object):
324
+ """The whole profile."""
325
+
326
+ def __init__(self):
327
+ Object.__init__(self)
328
+ self.functions = {}
329
+ self.cycles = []
330
+
331
+ def add_function(self, function):
332
+ if function.id in self.functions:
333
+ sys.stderr.write(
334
+ "warning: overwriting function %s (id %s)\n"
335
+ % (function.name, str(function.id))
336
+ )
337
+ self.functions[function.id] = function
338
+
339
+ def add_cycle(self, cycle):
340
+ self.cycles.append(cycle)
341
+
342
+ def validate(self):
343
+ """Validate the edges."""
344
+
345
+ for function in compat_itervalues(self.functions):
346
+ for callee_id in compat_keys(function.calls):
347
+ assert function.calls[callee_id].callee_id == callee_id
348
+ if callee_id not in self.functions:
349
+ sys.stderr.write(
350
+ "warning: call to undefined function %s from function %s\n"
351
+ % (str(callee_id), function.name)
352
+ )
353
+ del function.calls[callee_id]
354
+
355
+ def find_cycles(self):
356
+ """Find cycles using Tarjan's strongly connected components algorithm."""
357
+
358
+ # Apply the Tarjan's algorithm successively until all functions are visited
359
+ stack = []
360
+ data = {}
361
+ order = 0
362
+ for function in compat_itervalues(self.functions):
363
+ order = self._tarjan(function, order, stack, data)
364
+ cycles = []
365
+ for function in compat_itervalues(self.functions):
366
+ if function.cycle is not None and function.cycle not in cycles:
367
+ cycles.append(function.cycle)
368
+ self.cycles = cycles
369
+ if 0:
370
+ for cycle in cycles:
371
+ sys.stderr.write("Cycle:\n")
372
+ for member in cycle.functions:
373
+ sys.stderr.write("\tFunction %s\n" % member.name)
374
+
375
+ def prune_root(self, roots, depth=-1):
376
+ visited = set()
377
+ frontier = set([(root_node, depth) for root_node in roots])
378
+ while len(frontier) > 0:
379
+ node, node_depth = frontier.pop()
380
+ visited.add(node)
381
+ if node_depth == 0:
382
+ continue
383
+ f = self.functions[node]
384
+ newNodes = set(f.calls.keys()) - visited
385
+ frontier = frontier.union(
386
+ {(new_node, node_depth - 1) for new_node in newNodes}
387
+ )
388
+ subtreeFunctions = {}
389
+ for n in visited:
390
+ f = self.functions[n]
391
+ newCalls = {}
392
+ for c in f.calls.keys():
393
+ if c in visited:
394
+ newCalls[c] = f.calls[c]
395
+ f.calls = newCalls
396
+ subtreeFunctions[n] = f
397
+ self.functions = subtreeFunctions
398
+
399
+ def prune_leaf(self, leafs, depth=-1):
400
+ edgesUp = collections.defaultdict(set)
401
+ for f in self.functions.keys():
402
+ for n in self.functions[f].calls.keys():
403
+ edgesUp[n].add(f)
404
+ # build the tree up
405
+ visited = set()
406
+ frontier = set([(leaf_node, depth) for leaf_node in leafs])
407
+ while len(frontier) > 0:
408
+ node, node_depth = frontier.pop()
409
+ visited.add(node)
410
+ if node_depth == 0:
411
+ continue
412
+ newNodes = edgesUp[node] - visited
413
+ frontier = frontier.union(
414
+ {(new_node, node_depth - 1) for new_node in newNodes}
415
+ )
416
+ downTree = set(self.functions.keys())
417
+ upTree = visited
418
+ path = downTree.intersection(upTree)
419
+ pathFunctions = {}
420
+ for n in path:
421
+ f = self.functions[n]
422
+ newCalls = {}
423
+ for c in f.calls.keys():
424
+ if c in path:
425
+ newCalls[c] = f.calls[c]
426
+ f.calls = newCalls
427
+ pathFunctions[n] = f
428
+ self.functions = pathFunctions
429
+
430
+ def getFunctionIds(self, funcName):
431
+ function_names = {v.name: k for (k, v) in self.functions.items()}
432
+ return [
433
+ function_names[name]
434
+ for name in fnmatch.filter(function_names.keys(), funcName)
435
+ ]
436
+
437
+ def getFunctionId(self, funcName):
438
+ for f in self.functions:
439
+ if self.functions[f].name == funcName:
440
+ return f
441
+ return False
442
+
443
+ def printFunctionIds(self, selector=None, file=sys.stderr):
444
+ """Print to file function entries selected by fnmatch.fnmatch like in
445
+ method getFunctionIds, with following extensions:
446
+ - selector starts with "%": dump all information available
447
+ - selector is '+' or '-': select all function entries
448
+ """
449
+ if selector is None or selector in ("+", "*"):
450
+ v = ",\n".join(
451
+ (
452
+ "%s:\t%s" % (kf, self.functions[kf].name)
453
+ for kf in self.functions.keys()
454
+ )
455
+ )
456
+ else:
457
+ if selector[0] == "%":
458
+ selector = selector[1:]
459
+ function_info = {
460
+ k: v
461
+ for (k, v) in self.functions.items()
462
+ if fnmatch.fnmatch(v.name, selector)
463
+ }
464
+ v = ",\n".join(
465
+ (
466
+ "%s\t({k})\t(%s)::\n\t%s" % (v.name, type(v), v.dump())
467
+ for (k, v) in function_info.items()
468
+ )
469
+ )
470
+
471
+ else:
472
+ function_names = (v.name for v in self.functions.values())
473
+ v = ",\n".join((nm for nm in fnmatch.filter(function_names, selector)))
474
+
475
+ file.write(v + "\n")
476
+ file.flush()
477
+
478
+ class _TarjanData:
479
+ def __init__(self, order):
480
+ self.order = order
481
+ self.lowlink = order
482
+ self.onstack = False
483
+
484
+ def _tarjan(self, function, order, stack, data):
485
+ """Tarjan's strongly connected components algorithm.
486
+
487
+ See also:
488
+ - http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm
489
+ """
490
+
491
+ try:
492
+ func_data = data[function.id]
493
+ return order
494
+ except KeyError:
495
+ func_data = self._TarjanData(order)
496
+ data[function.id] = func_data
497
+ order += 1
498
+ pos = len(stack)
499
+ stack.append(function)
500
+ func_data.onstack = True
501
+ for call in compat_itervalues(function.calls):
502
+ try:
503
+ callee_data = data[call.callee_id]
504
+ if callee_data.onstack:
505
+ func_data.lowlink = min(func_data.lowlink, callee_data.order)
506
+ except KeyError:
507
+ callee = self.functions[call.callee_id]
508
+ order = self._tarjan(callee, order, stack, data)
509
+ callee_data = data[call.callee_id]
510
+ func_data.lowlink = min(func_data.lowlink, callee_data.lowlink)
511
+ if func_data.lowlink == func_data.order:
512
+ # Strongly connected component found
513
+ members = stack[pos:]
514
+ del stack[pos:]
515
+ if len(members) > 1:
516
+ cycle = Cycle()
517
+ for member in members:
518
+ cycle.add_function(member)
519
+ data[member.id].onstack = False
520
+ else:
521
+ for member in members:
522
+ data[member.id].onstack = False
523
+ return order
524
+
525
+ def call_ratios(self, event):
526
+ # Aggregate for incoming calls
527
+ cycle_totals = {}
528
+ for cycle in self.cycles:
529
+ cycle_totals[cycle] = 0.0
530
+ function_totals = {}
531
+ for function in compat_itervalues(self.functions):
532
+ function_totals[function] = 0.0
533
+
534
+ # Pass 1: function_total gets the sum of call[event] for all
535
+ # incoming arrows. Same for cycle_total for all arrows
536
+ # that are coming into the *cycle* but are not part of it.
537
+ for function in compat_itervalues(self.functions):
538
+ for call in compat_itervalues(function.calls):
539
+ if call.callee_id != function.id:
540
+ callee = self.functions[call.callee_id]
541
+ if event in call.events:
542
+ function_totals[callee] += call[event]
543
+ if (
544
+ callee.cycle is not None
545
+ and callee.cycle is not function.cycle
546
+ ):
547
+ cycle_totals[callee.cycle] += call[event]
548
+ else:
549
+ sys.stderr.write(
550
+ "call_ratios: No data for "
551
+ + function.name
552
+ + " call to "
553
+ + callee.name
554
+ + "\n"
555
+ )
556
+
557
+ # Pass 2: Compute the ratios. Each call[event] is scaled by the
558
+ # function_total of the callee. Calls into cycles use the
559
+ # cycle_total, but not calls within cycles.
560
+ for function in compat_itervalues(self.functions):
561
+ for call in compat_itervalues(function.calls):
562
+ assert call.ratio is None
563
+ if call.callee_id != function.id:
564
+ callee = self.functions[call.callee_id]
565
+ if event in call.events:
566
+ if (
567
+ callee.cycle is not None
568
+ and callee.cycle is not function.cycle
569
+ ):
570
+ total = cycle_totals[callee.cycle]
571
+ else:
572
+ total = function_totals[callee]
573
+ call.ratio = ratio(call[event], total)
574
+ else:
575
+ # Warnings here would only repeat those issued above.
576
+ call.ratio = 0.0
577
+
578
+ def integrate(self, outevent, inevent):
579
+ """Propagate function time ratio along the function calls.
580
+
581
+ Must be called after finding the cycles.
582
+
583
+ See also:
584
+ - http://citeseer.ist.psu.edu/graham82gprof.html
585
+ """
586
+
587
+ # Sanity checking
588
+ assert outevent not in self
589
+ for function in compat_itervalues(self.functions):
590
+ assert outevent not in function
591
+ assert inevent in function
592
+ for call in compat_itervalues(function.calls):
593
+ assert outevent not in call
594
+ if call.callee_id != function.id:
595
+ assert call.ratio is not None
596
+
597
+ # Aggregate the input for each cycle
598
+ for cycle in self.cycles:
599
+ total = inevent.null()
600
+ for function in compat_itervalues(self.functions):
601
+ total = inevent.aggregate(total, function[inevent])
602
+ self[inevent] = total
603
+
604
+ # Integrate along the edges
605
+ total = inevent.null()
606
+ for function in compat_itervalues(self.functions):
607
+ total = inevent.aggregate(total, function[inevent])
608
+ self._integrate_function(function, outevent, inevent)
609
+ self[outevent] = total
610
+
611
+ def _integrate_function(self, function, outevent, inevent):
612
+ if function.cycle is not None:
613
+ return self._integrate_cycle(function.cycle, outevent, inevent)
614
+ else:
615
+ if outevent not in function:
616
+ total = function[inevent]
617
+ for call in compat_itervalues(function.calls):
618
+ if call.callee_id != function.id:
619
+ total += self._integrate_call(call, outevent, inevent)
620
+ function[outevent] = total
621
+ return function[outevent]
622
+
623
+ def _integrate_call(self, call, outevent, inevent):
624
+ assert outevent not in call
625
+ assert call.ratio is not None
626
+ callee = self.functions[call.callee_id]
627
+ subtotal = call.ratio * self._integrate_function(callee, outevent, inevent)
628
+ call[outevent] = subtotal
629
+ return subtotal
630
+
631
+ def _integrate_cycle(self, cycle, outevent, inevent):
632
+ if outevent not in cycle:
633
+ # Compute the outevent for the whole cycle
634
+ total = inevent.null()
635
+ for member in cycle.functions:
636
+ subtotal = member[inevent]
637
+ for call in compat_itervalues(member.calls):
638
+ callee = self.functions[call.callee_id]
639
+ if callee.cycle is not cycle:
640
+ subtotal += self._integrate_call(call, outevent, inevent)
641
+ total += subtotal
642
+ cycle[outevent] = total
643
+
644
+ # Compute the time propagated to callers of this cycle
645
+ callees = {}
646
+ for function in compat_itervalues(self.functions):
647
+ if function.cycle is not cycle:
648
+ for call in compat_itervalues(function.calls):
649
+ callee = self.functions[call.callee_id]
650
+ if callee.cycle is cycle:
651
+ try:
652
+ callees[callee] += call.ratio
653
+ except KeyError:
654
+ callees[callee] = call.ratio
655
+
656
+ for member in cycle.functions:
657
+ member[outevent] = outevent.null()
658
+
659
+ for callee, call_ratio in compat_iteritems(callees):
660
+ ranks = {}
661
+ call_ratios = {}
662
+ partials = {}
663
+ self._rank_cycle_function(cycle, callee, ranks)
664
+ self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set())
665
+ partial = self._integrate_cycle_function(
666
+ cycle,
667
+ callee,
668
+ call_ratio,
669
+ partials,
670
+ ranks,
671
+ call_ratios,
672
+ outevent,
673
+ inevent,
674
+ )
675
+
676
+ # Ensure `partial == max(partials.values())`, but with round-off tolerance
677
+ max_partial = max(partials.values())
678
+ assert abs(partial - max_partial) <= 1e-7 * max_partial
679
+
680
+ assert abs(call_ratio * total - partial) <= 0.001 * call_ratio * total
681
+
682
+ return cycle[outevent]
683
+
684
+ def _rank_cycle_function(self, cycle, function, ranks):
685
+ """Dijkstra's shortest paths algorithm.
686
+
687
+ See also:
688
+ - http://en.wikipedia.org/wiki/Dijkstra's_algorithm
689
+ """
690
+
691
+ import heapq
692
+
693
+ Q = []
694
+ Qd = {}
695
+ p = {}
696
+ visited = set([function])
697
+
698
+ ranks[function] = 0
699
+ for call in compat_itervalues(function.calls):
700
+ if call.callee_id != function.id:
701
+ callee = self.functions[call.callee_id]
702
+ if callee.cycle is cycle:
703
+ ranks[callee] = 1
704
+ item = [ranks[callee], function, callee]
705
+ heapq.heappush(Q, item)
706
+ Qd[callee] = item
707
+
708
+ while Q:
709
+ cost, parent, member = heapq.heappop(Q)
710
+ if member not in visited:
711
+ p[member] = parent
712
+ visited.add(member)
713
+ for call in compat_itervalues(member.calls):
714
+ if call.callee_id != member.id:
715
+ callee = self.functions[call.callee_id]
716
+ if callee.cycle is cycle:
717
+ member_rank = ranks[member]
718
+ rank = ranks.get(callee)
719
+ if rank is not None:
720
+ if rank > 1 + member_rank:
721
+ rank = 1 + member_rank
722
+ ranks[callee] = rank
723
+ Qd_callee = Qd[callee]
724
+ Qd_callee[0] = rank
725
+ Qd_callee[1] = member
726
+ heapq._siftdown(Q, 0, Q.index(Qd_callee))
727
+ else:
728
+ rank = 1 + member_rank
729
+ ranks[callee] = rank
730
+ item = [rank, member, callee]
731
+ heapq.heappush(Q, item)
732
+ Qd[callee] = item
733
+
734
+ def _call_ratios_cycle(self, cycle, function, ranks, call_ratios, visited):
735
+ if function not in visited:
736
+ visited.add(function)
737
+ for call in compat_itervalues(function.calls):
738
+ if call.callee_id != function.id:
739
+ callee = self.functions[call.callee_id]
740
+ if callee.cycle is cycle:
741
+ if ranks[callee] > ranks[function]:
742
+ call_ratios[callee] = (
743
+ call_ratios.get(callee, 0.0) + call.ratio
744
+ )
745
+ self._call_ratios_cycle(
746
+ cycle, callee, ranks, call_ratios, visited
747
+ )
748
+
749
+ def _integrate_cycle_function(
750
+ self,
751
+ cycle,
752
+ function,
753
+ partial_ratio,
754
+ partials,
755
+ ranks,
756
+ call_ratios,
757
+ outevent,
758
+ inevent,
759
+ ):
760
+ if function not in partials:
761
+ partial = partial_ratio * function[inevent]
762
+ for call in compat_itervalues(function.calls):
763
+ if call.callee_id != function.id:
764
+ callee = self.functions[call.callee_id]
765
+ if callee.cycle is not cycle:
766
+ assert outevent in call
767
+ partial += partial_ratio * call[outevent]
768
+ else:
769
+ if ranks[callee] > ranks[function]:
770
+ callee_partial = self._integrate_cycle_function(
771
+ cycle,
772
+ callee,
773
+ partial_ratio,
774
+ partials,
775
+ ranks,
776
+ call_ratios,
777
+ outevent,
778
+ inevent,
779
+ )
780
+ call_ratio = ratio(call.ratio, call_ratios[callee])
781
+ call_partial = call_ratio * callee_partial
782
+ try:
783
+ call[outevent] += call_partial
784
+ except UndefinedEvent:
785
+ call[outevent] = call_partial
786
+ partial += call_partial
787
+ partials[function] = partial
788
+ try:
789
+ function[outevent] += partial
790
+ except UndefinedEvent:
791
+ function[outevent] = partial
792
+ return partials[function]
793
+
794
+ def aggregate(self, event):
795
+ """Aggregate an event for the whole profile."""
796
+
797
+ total = event.null()
798
+ for function in compat_itervalues(self.functions):
799
+ try:
800
+ total = event.aggregate(total, function[event])
801
+ except UndefinedEvent:
802
+ return
803
+ self[event] = total
804
+
805
+ def ratio(self, outevent, inevent):
806
+ assert outevent not in self
807
+ assert inevent in self
808
+ for function in compat_itervalues(self.functions):
809
+ assert outevent not in function
810
+ assert inevent in function
811
+ function[outevent] = ratio(function[inevent], self[inevent])
812
+ for call in compat_itervalues(function.calls):
813
+ assert outevent not in call
814
+ if inevent in call:
815
+ call[outevent] = ratio(call[inevent], self[inevent])
816
+ self[outevent] = 1.0
817
+
818
+ def prune(self, node_thres, edge_thres, paths, color_nodes_by_selftime):
819
+ """Prune the profile"""
820
+
821
+ # compute the prune ratios
822
+ for function in compat_itervalues(self.functions):
823
+ try:
824
+ function.weight = function[TOTAL_TIME_RATIO]
825
+ except UndefinedEvent:
826
+ pass
827
+
828
+ for call in compat_itervalues(function.calls):
829
+ callee = self.functions[call.callee_id]
830
+
831
+ if TOTAL_TIME_RATIO in call:
832
+ # handle exact cases first
833
+ call.weight = call[TOTAL_TIME_RATIO]
834
+ else:
835
+ try:
836
+ # make a safe estimate
837
+ call.weight = min(
838
+ function[TOTAL_TIME_RATIO], callee[TOTAL_TIME_RATIO]
839
+ )
840
+ except UndefinedEvent:
841
+ pass
842
+
843
+ # prune the nodes
844
+ for function_id in compat_keys(self.functions):
845
+ function = self.functions[function_id]
846
+ if function.weight is not None:
847
+ if function.weight < node_thres:
848
+ del self.functions[function_id]
849
+
850
+ # prune file paths
851
+ for function_id in compat_keys(self.functions):
852
+ function = self.functions[function_id]
853
+ if (
854
+ paths
855
+ and function.filename
856
+ and not any(function.filename.startswith(path) for path in paths)
857
+ ):
858
+ del self.functions[function_id]
859
+ elif (
860
+ paths
861
+ and function.module
862
+ and not any((function.module.find(path) > -1) for path in paths)
863
+ ):
864
+ del self.functions[function_id]
865
+
866
+ # prune the edges
867
+ for function in compat_itervalues(self.functions):
868
+ for callee_id in compat_keys(function.calls):
869
+ call = function.calls[callee_id]
870
+ if (
871
+ callee_id not in self.functions
872
+ or call.weight is not None
873
+ and call.weight < edge_thres
874
+ ):
875
+ del function.calls[callee_id]
876
+
877
+ if color_nodes_by_selftime:
878
+ weights = []
879
+ for function in compat_itervalues(self.functions):
880
+ try:
881
+ weights.append(function[TIME_RATIO])
882
+ except UndefinedEvent:
883
+ pass
884
+ max_ratio = max(weights or [1])
885
+
886
+ # apply rescaled weights for coloriung
887
+ for function in compat_itervalues(self.functions):
888
+ try:
889
+ function.weight = function[TIME_RATIO] / max_ratio
890
+ except (ZeroDivisionError, UndefinedEvent):
891
+ pass
892
+
893
+ def dump(self):
894
+ for function in compat_itervalues(self.functions):
895
+ sys.stderr.write("Function %s:\n" % (function.name,))
896
+ self._dump_events(function.events)
897
+ for call in compat_itervalues(function.calls):
898
+ callee = self.functions[call.callee_id]
899
+ sys.stderr.write(" Call %s:\n" % (callee.name,))
900
+ self._dump_events(call.events)
901
+ for cycle in self.cycles:
902
+ sys.stderr.write("Cycle:\n")
903
+ self._dump_events(cycle.events)
904
+ for function in cycle.functions:
905
+ sys.stderr.write(" Function %s\n" % (function.name,))
906
+
907
+ def _dump_events(self, events):
908
+ for event, value in compat_iteritems(events):
909
+ sys.stderr.write(" %s: %s\n" % (event.name, event.format(value)))
910
+
911
+
912
+ ########################################################################
913
+ # Parsers
914
+
915
+
916
+ class Struct:
917
+ """Masquerade a dictionary with a structure-like behavior."""
918
+
919
+ def __init__(self, attrs=None):
920
+ if attrs is None:
921
+ attrs = {}
922
+ self.__dict__["_attrs"] = attrs
923
+
924
+ def __getattr__(self, name):
925
+ try:
926
+ return self._attrs[name]
927
+ except KeyError:
928
+ raise AttributeError(name)
929
+
930
+ def __setattr__(self, name, value):
931
+ self._attrs[name] = value
932
+
933
+ def __str__(self):
934
+ return str(self._attrs)
935
+
936
+ def __repr__(self):
937
+ return repr(self._attrs)
938
+
939
+
940
+ class ParseError(Exception):
941
+ """Raised when parsing to signal mismatches."""
942
+
943
+ def __init__(self, msg, line):
944
+ Exception.__init__(self)
945
+ self.msg = msg
946
+ # TODO: store more source line information
947
+ self.line = line
948
+
949
+ def __str__(self):
950
+ return "%s: %r" % (self.msg, self.line)
951
+
952
+
953
+ class Parser:
954
+ """Parser interface."""
955
+
956
+ stdinInput = True
957
+ multipleInput = False
958
+
959
+ def __init__(self):
960
+ pass
961
+
962
+ def parse(self):
963
+ raise NotImplementedError
964
+
965
+
966
+ class JsonParser(Parser):
967
+ """Parser for a custom JSON representation of profile data.
968
+
969
+ See schema.json for details.
970
+ """
971
+
972
+ def __init__(self, stream):
973
+ Parser.__init__(self)
974
+ self.stream = stream
975
+
976
+ def parse(self):
977
+ obj = json.load(self.stream)
978
+
979
+ assert obj["version"] == 0
980
+
981
+ profile = Profile()
982
+ profile[SAMPLES] = 0
983
+
984
+ fns = obj["functions"]
985
+
986
+ for functionIndex in range(len(fns)):
987
+ fn = fns[functionIndex]
988
+ function = Function(functionIndex, fn["name"])
989
+ try:
990
+ function.module = fn["module"]
991
+ except KeyError:
992
+ pass
993
+ try:
994
+ function.process = fn["process"]
995
+ except KeyError:
996
+ pass
997
+ function[SAMPLES] = 0
998
+ function.called = 0
999
+ profile.add_function(function)
1000
+
1001
+ for event in obj["events"]:
1002
+ callchain = []
1003
+
1004
+ for functionIndex in event["callchain"]:
1005
+ function = profile.functions[functionIndex]
1006
+ callchain.append(function)
1007
+
1008
+ # increment the call count of the first in the callchain
1009
+ function = profile.functions[event["callchain"][0]]
1010
+ function.called = function.called + 1
1011
+
1012
+ cost = event["cost"][0]
1013
+
1014
+ callee = callchain[0]
1015
+ callee[SAMPLES] += cost
1016
+ profile[SAMPLES] += cost
1017
+
1018
+ for caller in callchain[1:]:
1019
+ try:
1020
+ call = caller.calls[callee.id]
1021
+ except KeyError:
1022
+ call = Call(callee.id)
1023
+ call[SAMPLES2] = cost
1024
+ caller.add_call(call)
1025
+ else:
1026
+ call[SAMPLES2] += cost
1027
+
1028
+ callee = caller
1029
+
1030
+ if False:
1031
+ profile.dump()
1032
+
1033
+ # compute derived data
1034
+ profile.validate()
1035
+ profile.find_cycles()
1036
+ profile.ratio(TIME_RATIO, SAMPLES)
1037
+ profile.call_ratios(SAMPLES2)
1038
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
1039
+
1040
+ return profile
1041
+
1042
+
1043
+ class LineParser(Parser):
1044
+ """Base class for parsers that read line-based formats."""
1045
+
1046
+ def __init__(self, stream):
1047
+ Parser.__init__(self)
1048
+ self._stream = stream
1049
+ self.__line = None
1050
+ self.__eof = False
1051
+ self.line_no = 0
1052
+
1053
+ def readline(self):
1054
+ line = self._stream.readline()
1055
+ if not line:
1056
+ self.__line = ""
1057
+ self.__eof = True
1058
+ else:
1059
+ self.line_no += 1
1060
+ line = line.rstrip("\r\n")
1061
+ if not PYTHON_3:
1062
+ encoding = self._stream.encoding
1063
+ if encoding is None:
1064
+ encoding = locale.getpreferredencoding()
1065
+ line = line.decode(encoding)
1066
+ self.__line = line
1067
+
1068
+ def lookahead(self):
1069
+ assert self.__line is not None
1070
+ return self.__line
1071
+
1072
+ def consume(self):
1073
+ assert self.__line is not None
1074
+ line = self.__line
1075
+ self.readline()
1076
+ return line
1077
+
1078
+ def eof(self):
1079
+ assert self.__line is not None
1080
+ return self.__eof
1081
+
1082
+
1083
+ XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF = range(4)
1084
+
1085
+
1086
+ class XmlToken:
1087
+ def __init__(self, type, name_or_data, attrs=None, line=None, column=None):
1088
+ assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF)
1089
+ self.type = type
1090
+ self.name_or_data = name_or_data
1091
+ self.attrs = attrs
1092
+ self.line = line
1093
+ self.column = column
1094
+
1095
+ def __str__(self):
1096
+ if self.type == XML_ELEMENT_START:
1097
+ return "<" + self.name_or_data + " ...>"
1098
+ if self.type == XML_ELEMENT_END:
1099
+ return "</" + self.name_or_data + ">"
1100
+ if self.type == XML_CHARACTER_DATA:
1101
+ return self.name_or_data
1102
+ if self.type == XML_EOF:
1103
+ return "end of file"
1104
+ assert 0
1105
+
1106
+
1107
+ class XmlTokenizer:
1108
+ """Expat based XML tokenizer."""
1109
+
1110
+ def __init__(self, fp, skip_ws=True):
1111
+ self.fp = fp
1112
+ self.tokens = []
1113
+ self.index = 0
1114
+ self.final = False
1115
+ self.skip_ws = skip_ws
1116
+
1117
+ self.character_pos = 0, 0
1118
+ self.character_data = ""
1119
+
1120
+ self.parser = xml.parsers.expat.ParserCreate()
1121
+ self.parser.StartElementHandler = self.handle_element_start
1122
+ self.parser.EndElementHandler = self.handle_element_end
1123
+ self.parser.CharacterDataHandler = self.handle_character_data
1124
+
1125
+ def handle_element_start(self, name, attributes):
1126
+ self.finish_character_data()
1127
+ line, column = self.pos()
1128
+ token = XmlToken(XML_ELEMENT_START, name, attributes, line, column)
1129
+ self.tokens.append(token)
1130
+
1131
+ def handle_element_end(self, name):
1132
+ self.finish_character_data()
1133
+ line, column = self.pos()
1134
+ token = XmlToken(XML_ELEMENT_END, name, None, line, column)
1135
+ self.tokens.append(token)
1136
+
1137
+ def handle_character_data(self, data):
1138
+ if not self.character_data:
1139
+ self.character_pos = self.pos()
1140
+ self.character_data += data
1141
+
1142
+ def finish_character_data(self):
1143
+ if self.character_data:
1144
+ if not self.skip_ws or not self.character_data.isspace():
1145
+ line, column = self.character_pos
1146
+ token = XmlToken(
1147
+ XML_CHARACTER_DATA, self.character_data, None, line, column
1148
+ )
1149
+ self.tokens.append(token)
1150
+ self.character_data = ""
1151
+
1152
+ def next(self):
1153
+ size = 16 * 1024
1154
+ while self.index >= len(self.tokens) and not self.final:
1155
+ self.tokens = []
1156
+ self.index = 0
1157
+ data = self.fp.read(size)
1158
+ self.final = len(data) < size
1159
+ self.parser.Parse(data, self.final)
1160
+ if self.index >= len(self.tokens):
1161
+ line, column = self.pos()
1162
+ token = XmlToken(XML_EOF, None, None, line, column)
1163
+ else:
1164
+ token = self.tokens[self.index]
1165
+ self.index += 1
1166
+ return token
1167
+
1168
+ def pos(self):
1169
+ return self.parser.CurrentLineNumber, self.parser.CurrentColumnNumber
1170
+
1171
+
1172
+ class XmlTokenMismatch(Exception):
1173
+ def __init__(self, expected, found):
1174
+ Exception.__init__(self)
1175
+ self.expected = expected
1176
+ self.found = found
1177
+
1178
+ def __str__(self):
1179
+ return "%u:%u: %s expected, %s found" % (
1180
+ self.found.line,
1181
+ self.found.column,
1182
+ str(self.expected),
1183
+ str(self.found),
1184
+ )
1185
+
1186
+
1187
+ class XmlParser(Parser):
1188
+ """Base XML document parser."""
1189
+
1190
+ def __init__(self, fp):
1191
+ Parser.__init__(self)
1192
+ self.tokenizer = XmlTokenizer(fp)
1193
+ self.consume()
1194
+
1195
+ def consume(self):
1196
+ self.token = self.tokenizer.next()
1197
+
1198
+ def match_element_start(self, name):
1199
+ return self.token.type == XML_ELEMENT_START and self.token.name_or_data == name
1200
+
1201
+ def match_element_end(self, name):
1202
+ return self.token.type == XML_ELEMENT_END and self.token.name_or_data == name
1203
+
1204
+ def element_start(self, name):
1205
+ while self.token.type == XML_CHARACTER_DATA:
1206
+ self.consume()
1207
+ if self.token.type != XML_ELEMENT_START:
1208
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
1209
+ if self.token.name_or_data != name:
1210
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
1211
+ attrs = self.token.attrs
1212
+ self.consume()
1213
+ return attrs
1214
+
1215
+ def element_end(self, name):
1216
+ while self.token.type == XML_CHARACTER_DATA:
1217
+ self.consume()
1218
+ if self.token.type != XML_ELEMENT_END:
1219
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
1220
+ if self.token.name_or_data != name:
1221
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
1222
+ self.consume()
1223
+
1224
+ def character_data(self, strip=True):
1225
+ data = ""
1226
+ while self.token.type == XML_CHARACTER_DATA:
1227
+ data += self.token.name_or_data
1228
+ self.consume()
1229
+ if strip:
1230
+ data = data.strip()
1231
+ return data
1232
+
1233
+
1234
+ class GprofParser(Parser):
1235
+ """Parser for GNU gprof output.
1236
+
1237
+ See also:
1238
+ - Chapter "Interpreting gprof's Output" from the GNU gprof manual
1239
+ http://sourceware.org/binutils/docs-2.18/gprof/Call-Graph.html#Call-Graph
1240
+ - File "cg_print.c" from the GNU gprof source code
1241
+ http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/src/gprof/cg_print.c?rev=1.12&cvsroot=src
1242
+ """
1243
+
1244
+ def __init__(self, fp):
1245
+ Parser.__init__(self)
1246
+ self.fp = fp
1247
+ self.functions = {}
1248
+ self.cycles = {}
1249
+
1250
+ def readline(self):
1251
+ line = self.fp.readline()
1252
+ if not line:
1253
+ sys.stderr.write("error: unexpected end of file\n")
1254
+ sys.exit(1)
1255
+ line = line.rstrip("\r\n")
1256
+ return line
1257
+
1258
+ _int_re = re.compile(r"^\d+$")
1259
+ _float_re = re.compile(r"^\d+\.\d+$")
1260
+
1261
+ def translate(self, mo):
1262
+ """Extract a structure from a match object, while translating the types in the process."""
1263
+ attrs = {}
1264
+ groupdict = mo.groupdict()
1265
+ for name, value in compat_iteritems(groupdict):
1266
+ if value is None:
1267
+ value = None
1268
+ elif self._int_re.match(value):
1269
+ value = int(value)
1270
+ elif self._float_re.match(value):
1271
+ value = float(value)
1272
+ attrs[name] = value
1273
+ return Struct(attrs)
1274
+
1275
+ _cg_header_re = re.compile(
1276
+ # original gprof header
1277
+ r"^\s+called/total\s+parents\s*$|"
1278
+ + r"^index\s+%time\s+self\s+descendents\s+called\+self\s+name\s+index\s*$|"
1279
+ + r"^\s+called/total\s+children\s*$|"
1280
+ +
1281
+ # GNU gprof header
1282
+ r"^index\s+%\s+time\s+self\s+children\s+called\s+name\s*$"
1283
+ )
1284
+
1285
+ _cg_ignore_re = re.compile(
1286
+ # spontaneous
1287
+ r"^\s+<spontaneous>\s*$|"
1288
+ # internal calls (such as "mcount")
1289
+ r"^.*\((\d+)\)$"
1290
+ )
1291
+
1292
+ _cg_primary_re = re.compile(
1293
+ r"^\[(?P<index>\d+)\]?"
1294
+ + r"\s+(?P<percentage_time>\d+\.\d+)"
1295
+ + r"\s+(?P<self>\d+\.\d+)"
1296
+ + r"\s+(?P<descendants>\d+\.\d+)"
1297
+ + r"\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?"
1298
+ + r"\s+(?P<name>\S.*?)"
1299
+ + r"(?:\s+<cycle\s(?P<cycle>\d+)>)?"
1300
+ + r"\s\[(\d+)\]$"
1301
+ )
1302
+
1303
+ _cg_parent_re = re.compile(
1304
+ r"^\s+(?P<self>\d+\.\d+)?"
1305
+ + r"\s+(?P<descendants>\d+\.\d+)?"
1306
+ + r"\s+(?P<called>\d+)(?:/(?P<called_total>\d+))?"
1307
+ + r"\s+(?P<name>\S.*?)"
1308
+ + r"(?:\s+<cycle\s(?P<cycle>\d+)>)?"
1309
+ + r"\s\[(?P<index>\d+)\]$"
1310
+ )
1311
+
1312
+ _cg_child_re = _cg_parent_re
1313
+
1314
+ _cg_cycle_header_re = re.compile(
1315
+ r"^\[(?P<index>\d+)\]?"
1316
+ + r"\s+(?P<percentage_time>\d+\.\d+)"
1317
+ + r"\s+(?P<self>\d+\.\d+)"
1318
+ + r"\s+(?P<descendants>\d+\.\d+)"
1319
+ + r"\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?"
1320
+ + r"\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>"
1321
+ + r"\s\[(\d+)\]$"
1322
+ )
1323
+
1324
+ _cg_cycle_member_re = re.compile(
1325
+ r"^\s+(?P<self>\d+\.\d+)?"
1326
+ + r"\s+(?P<descendants>\d+\.\d+)?"
1327
+ + r"\s+(?P<called>\d+)(?:\+(?P<called_self>\d+))?"
1328
+ + r"\s+(?P<name>\S.*?)"
1329
+ + r"(?:\s+<cycle\s(?P<cycle>\d+)>)?"
1330
+ + r"\s\[(?P<index>\d+)\]$"
1331
+ )
1332
+
1333
+ _cg_sep_re = re.compile(r"^--+$")
1334
+
1335
+ def parse_function_entry(self, lines):
1336
+ parents = []
1337
+ children = []
1338
+
1339
+ while True:
1340
+ if not lines:
1341
+ sys.stderr.write("warning: unexpected end of entry\n")
1342
+ line = lines.pop(0)
1343
+ if line.startswith("["):
1344
+ break
1345
+
1346
+ # read function parent line
1347
+ mo = self._cg_parent_re.match(line)
1348
+ if not mo:
1349
+ if self._cg_ignore_re.match(line):
1350
+ continue
1351
+ sys.stderr.write("warning: unrecognized call graph entry: %r\n" % line)
1352
+ else:
1353
+ parent = self.translate(mo)
1354
+ parents.append(parent)
1355
+
1356
+ # read primary line
1357
+ mo = self._cg_primary_re.match(line)
1358
+ if not mo:
1359
+ sys.stderr.write("warning: unrecognized call graph entry: %r\n" % line)
1360
+ return
1361
+ else:
1362
+ function = self.translate(mo)
1363
+
1364
+ while lines:
1365
+ line = lines.pop(0)
1366
+
1367
+ # read function subroutine line
1368
+ mo = self._cg_child_re.match(line)
1369
+ if not mo:
1370
+ if self._cg_ignore_re.match(line):
1371
+ continue
1372
+ sys.stderr.write("warning: unrecognized call graph entry: %r\n" % line)
1373
+ else:
1374
+ child = self.translate(mo)
1375
+ children.append(child)
1376
+
1377
+ function.parents = parents
1378
+ function.children = children
1379
+
1380
+ self.functions[function.index] = function
1381
+
1382
+ def parse_cycle_entry(self, lines):
1383
+ # read cycle header line
1384
+ line = lines[0]
1385
+ mo = self._cg_cycle_header_re.match(line)
1386
+ if not mo:
1387
+ sys.stderr.write("warning: unrecognized call graph entry: %r\n" % line)
1388
+ return
1389
+ cycle = self.translate(mo)
1390
+
1391
+ # read cycle member lines
1392
+ cycle.functions = []
1393
+ for line in lines[1:]:
1394
+ mo = self._cg_cycle_member_re.match(line)
1395
+ if not mo:
1396
+ sys.stderr.write("warning: unrecognized call graph entry: %r\n" % line)
1397
+ continue
1398
+ call = self.translate(mo)
1399
+ cycle.functions.append(call)
1400
+
1401
+ self.cycles[cycle.cycle] = cycle
1402
+
1403
+ def parse_cg_entry(self, lines):
1404
+ if lines[0].startswith("["):
1405
+ self.parse_cycle_entry(lines)
1406
+ else:
1407
+ self.parse_function_entry(lines)
1408
+
1409
+ def parse_cg(self):
1410
+ """Parse the call graph."""
1411
+
1412
+ # skip call graph header
1413
+ while not self._cg_header_re.match(self.readline()):
1414
+ pass
1415
+ line = self.readline()
1416
+ while self._cg_header_re.match(line):
1417
+ line = self.readline()
1418
+
1419
+ # process call graph entries
1420
+ entry_lines = []
1421
+ while line != "\014": # form feed
1422
+ if line and not line.isspace():
1423
+ if self._cg_sep_re.match(line):
1424
+ self.parse_cg_entry(entry_lines)
1425
+ entry_lines = []
1426
+ else:
1427
+ entry_lines.append(line)
1428
+ line = self.readline()
1429
+
1430
+ def parse(self):
1431
+ self.parse_cg()
1432
+ self.fp.close()
1433
+
1434
+ profile = Profile()
1435
+ profile[TIME] = 0.0
1436
+
1437
+ cycles = {}
1438
+ for index in self.cycles:
1439
+ cycles[index] = Cycle()
1440
+
1441
+ for entry in compat_itervalues(self.functions):
1442
+ # populate the function
1443
+ function = Function(entry.index, entry.name)
1444
+ function[TIME] = entry.self
1445
+ if entry.called is not None:
1446
+ function.called = entry.called
1447
+ if entry.called_self is not None:
1448
+ call = Call(entry.index)
1449
+ call[CALLS] = entry.called_self
1450
+ function.called += entry.called_self
1451
+
1452
+ # populate the function calls
1453
+ for child in entry.children:
1454
+ call = Call(child.index)
1455
+
1456
+ assert child.called is not None
1457
+ call[CALLS] = child.called
1458
+
1459
+ if child.index not in self.functions:
1460
+ # NOTE: functions that were never called but were discovered by gprof's
1461
+ # static call graph analysis dont have a call graph entry so we need
1462
+ # to add them here
1463
+ missing = Function(child.index, child.name)
1464
+ function[TIME] = 0.0
1465
+ function.called = 0
1466
+ profile.add_function(missing)
1467
+
1468
+ function.add_call(call)
1469
+
1470
+ profile.add_function(function)
1471
+
1472
+ if entry.cycle is not None:
1473
+ try:
1474
+ cycle = cycles[entry.cycle]
1475
+ except KeyError:
1476
+ sys.stderr.write(
1477
+ "warning: <cycle %u as a whole> entry missing\n" % entry.cycle
1478
+ )
1479
+ cycle = Cycle()
1480
+ cycles[entry.cycle] = cycle
1481
+ cycle.add_function(function)
1482
+
1483
+ profile[TIME] = profile[TIME] + function[TIME]
1484
+
1485
+ for cycle in compat_itervalues(cycles):
1486
+ profile.add_cycle(cycle)
1487
+
1488
+ # Compute derived events
1489
+ profile.validate()
1490
+ profile.ratio(TIME_RATIO, TIME)
1491
+ profile.call_ratios(CALLS)
1492
+ profile.integrate(TOTAL_TIME, TIME)
1493
+ profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
1494
+
1495
+ return profile
1496
+
1497
+
1498
+ # Clone&hack of GprofParser for VTune Amplifier XE 2013 gprof-cc output.
1499
+ # Tested only with AXE 2013 for Windows.
1500
+ # - Use total times as reported by AXE.
1501
+ # - In the absence of call counts, call ratios are faked from the relative
1502
+ # proportions of total time. This affects only the weighting of the calls.
1503
+ # - Different header, separator, and end marker.
1504
+ # - Extra whitespace after function names.
1505
+ # - You get a full entry for <spontaneous>, which does not have parents.
1506
+ # - Cycles do have parents. These are saved but unused (as they are
1507
+ # for functions).
1508
+ # - Disambiguated "unrecognized call graph entry" error messages.
1509
+ # Notes:
1510
+ # - Total time of functions as reported by AXE passes the val3 test.
1511
+ # - CPU Time:Children in the input is sometimes a negative number. This
1512
+ # value goes to the variable descendants, which is unused.
1513
+ # - The format of gprof-cc reports is unaffected by the use of
1514
+ # -knob enable-call-counts=true (no call counts, ever), or
1515
+ # -show-as=samples (results are quoted in seconds regardless).
1516
+ class AXEParser(Parser):
1517
+ "Parser for VTune Amplifier XE 2013 gprof-cc report output."
1518
+
1519
+ def __init__(self, fp):
1520
+ Parser.__init__(self)
1521
+ self.fp = fp
1522
+ self.functions = {}
1523
+ self.cycles = {}
1524
+
1525
+ def readline(self):
1526
+ line = self.fp.readline()
1527
+ if not line:
1528
+ sys.stderr.write("error: unexpected end of file\n")
1529
+ sys.exit(1)
1530
+ line = line.rstrip("\r\n")
1531
+ return line
1532
+
1533
+ _int_re = re.compile(r"^\d+$")
1534
+ _float_re = re.compile(r"^\d+\.\d+$")
1535
+
1536
+ def translate(self, mo):
1537
+ """Extract a structure from a match object, while translating the types in the process."""
1538
+ attrs = {}
1539
+ groupdict = mo.groupdict()
1540
+ for name, value in compat_iteritems(groupdict):
1541
+ if value is None:
1542
+ value = None
1543
+ elif self._int_re.match(value):
1544
+ value = int(value)
1545
+ elif self._float_re.match(value):
1546
+ value = float(value)
1547
+ attrs[name] = value
1548
+ return Struct(attrs)
1549
+
1550
+ _cg_header_re = re.compile("^Index |" "^-----+ ")
1551
+
1552
+ _cg_footer_re = re.compile(r"^Index\s+Function\s*$")
1553
+
1554
+ _cg_primary_re = re.compile(
1555
+ r"^\[(?P<index>\d+)\]?"
1556
+ + r"\s+(?P<percentage_time>\d+\.\d+)"
1557
+ + r"\s+(?P<self>\d+\.\d+)"
1558
+ + r"\s+(?P<descendants>\d+\.\d+)"
1559
+ + r"\s+(?P<name>\S.*?)"
1560
+ + r"(?:\s+<cycle\s(?P<cycle>\d+)>)?"
1561
+ + r"\s+\[(\d+)\]"
1562
+ + r"\s*$"
1563
+ )
1564
+
1565
+ _cg_parent_re = re.compile(
1566
+ r"^\s+(?P<self>\d+\.\d+)?"
1567
+ + r"\s+(?P<descendants>\d+\.\d+)?"
1568
+ + r"\s+(?P<name>\S.*?)"
1569
+ + r"(?:\s+<cycle\s(?P<cycle>\d+)>)?"
1570
+ + r"(?:\s+\[(?P<index>\d+)\]\s*)?"
1571
+ + r"\s*$"
1572
+ )
1573
+
1574
+ _cg_child_re = _cg_parent_re
1575
+
1576
+ _cg_cycle_header_re = re.compile(
1577
+ r"^\[(?P<index>\d+)\]?"
1578
+ + r"\s+(?P<percentage_time>\d+\.\d+)"
1579
+ + r"\s+(?P<self>\d+\.\d+)"
1580
+ + r"\s+(?P<descendants>\d+\.\d+)"
1581
+ + r"\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>"
1582
+ + r"\s+\[(\d+)\]"
1583
+ + r"\s*$"
1584
+ )
1585
+
1586
+ _cg_cycle_member_re = re.compile(
1587
+ r"^\s+(?P<self>\d+\.\d+)?"
1588
+ + r"\s+(?P<descendants>\d+\.\d+)?"
1589
+ + r"\s+(?P<name>\S.*?)"
1590
+ + r"(?:\s+<cycle\s(?P<cycle>\d+)>)?"
1591
+ + r"\s+\[(?P<index>\d+)\]"
1592
+ + r"\s*$"
1593
+ )
1594
+
1595
+ def parse_function_entry(self, lines):
1596
+ parents = []
1597
+ children = []
1598
+
1599
+ while True:
1600
+ if not lines:
1601
+ sys.stderr.write("warning: unexpected end of entry\n")
1602
+ return
1603
+ line = lines.pop(0)
1604
+ if line.startswith("["):
1605
+ break
1606
+
1607
+ # read function parent line
1608
+ mo = self._cg_parent_re.match(line)
1609
+ if not mo:
1610
+ sys.stderr.write(
1611
+ "warning: unrecognized call graph entry (1): %r\n" % line
1612
+ )
1613
+ else:
1614
+ parent = self.translate(mo)
1615
+ if parent.name != "<spontaneous>":
1616
+ parents.append(parent)
1617
+
1618
+ # read primary line
1619
+ mo = self._cg_primary_re.match(line)
1620
+ if not mo:
1621
+ sys.stderr.write("warning: unrecognized call graph entry (2): %r\n" % line)
1622
+ return
1623
+ else:
1624
+ function = self.translate(mo)
1625
+
1626
+ while lines:
1627
+ line = lines.pop(0)
1628
+
1629
+ # read function subroutine line
1630
+ mo = self._cg_child_re.match(line)
1631
+ if not mo:
1632
+ sys.stderr.write(
1633
+ "warning: unrecognized call graph entry (3): %r\n" % line
1634
+ )
1635
+ else:
1636
+ child = self.translate(mo)
1637
+ if child.name != "<spontaneous>":
1638
+ children.append(child)
1639
+
1640
+ if function.name != "<spontaneous>":
1641
+ function.parents = parents
1642
+ function.children = children
1643
+
1644
+ self.functions[function.index] = function
1645
+
1646
+ def parse_cycle_entry(self, lines):
1647
+ # Process the parents that were not there in gprof format.
1648
+ parents = []
1649
+ while True:
1650
+ if not lines:
1651
+ sys.stderr.write("warning: unexpected end of cycle entry\n")
1652
+ return
1653
+ line = lines.pop(0)
1654
+ if line.startswith("["):
1655
+ break
1656
+ mo = self._cg_parent_re.match(line)
1657
+ if not mo:
1658
+ sys.stderr.write(
1659
+ "warning: unrecognized call graph entry (6): %r\n" % line
1660
+ )
1661
+ else:
1662
+ parent = self.translate(mo)
1663
+ if parent.name != "<spontaneous>":
1664
+ parents.append(parent)
1665
+
1666
+ # read cycle header line
1667
+ mo = self._cg_cycle_header_re.match(line)
1668
+ if not mo:
1669
+ sys.stderr.write("warning: unrecognized call graph entry (4): %r\n" % line)
1670
+ return
1671
+ cycle = self.translate(mo)
1672
+
1673
+ # read cycle member lines
1674
+ cycle.functions = []
1675
+ for line in lines[1:]:
1676
+ mo = self._cg_cycle_member_re.match(line)
1677
+ if not mo:
1678
+ sys.stderr.write(
1679
+ "warning: unrecognized call graph entry (5): %r\n" % line
1680
+ )
1681
+ continue
1682
+ call = self.translate(mo)
1683
+ cycle.functions.append(call)
1684
+
1685
+ cycle.parents = parents
1686
+ self.cycles[cycle.cycle] = cycle
1687
+
1688
+ def parse_cg_entry(self, lines):
1689
+ if any("as a whole" in linelooper for linelooper in lines):
1690
+ self.parse_cycle_entry(lines)
1691
+ else:
1692
+ self.parse_function_entry(lines)
1693
+
1694
+ def parse_cg(self):
1695
+ """Parse the call graph."""
1696
+
1697
+ # skip call graph header
1698
+ line = self.readline()
1699
+ while self._cg_header_re.match(line):
1700
+ line = self.readline()
1701
+
1702
+ # process call graph entries
1703
+ entry_lines = []
1704
+ # An EOF in readline terminates the program without returning.
1705
+ while not self._cg_footer_re.match(line):
1706
+ if line.isspace():
1707
+ self.parse_cg_entry(entry_lines)
1708
+ entry_lines = []
1709
+ else:
1710
+ entry_lines.append(line)
1711
+ line = self.readline()
1712
+
1713
+ def parse(self):
1714
+ sys.stderr.write(
1715
+ "warning: for axe format, edge weights are unreliable estimates derived from function total times.\n"
1716
+ )
1717
+ self.parse_cg()
1718
+ self.fp.close()
1719
+
1720
+ profile = Profile()
1721
+ profile[TIME] = 0.0
1722
+
1723
+ cycles = {}
1724
+ for index in self.cycles:
1725
+ cycles[index] = Cycle()
1726
+
1727
+ for entry in compat_itervalues(self.functions):
1728
+ # populate the function
1729
+ function = Function(entry.index, entry.name)
1730
+ function[TIME] = entry.self
1731
+ function[TOTAL_TIME_RATIO] = entry.percentage_time / 100.0
1732
+
1733
+ # populate the function calls
1734
+ for child in entry.children:
1735
+ call = Call(child.index)
1736
+ # The following bogus value affects only the weighting of
1737
+ # the calls.
1738
+ call[TOTAL_TIME_RATIO] = function[TOTAL_TIME_RATIO]
1739
+
1740
+ if child.index not in self.functions:
1741
+ # NOTE: functions that were never called but were discovered by gprof's
1742
+ # static call graph analysis dont have a call graph entry so we need
1743
+ # to add them here
1744
+ # FIXME: Is this applicable?
1745
+ missing = Function(child.index, child.name)
1746
+ function[TIME] = 0.0
1747
+ profile.add_function(missing)
1748
+
1749
+ function.add_call(call)
1750
+
1751
+ profile.add_function(function)
1752
+
1753
+ if entry.cycle is not None:
1754
+ try:
1755
+ cycle = cycles[entry.cycle]
1756
+ except KeyError:
1757
+ sys.stderr.write(
1758
+ "warning: <cycle %u as a whole> entry missing\n" % entry.cycle
1759
+ )
1760
+ cycle = Cycle()
1761
+ cycles[entry.cycle] = cycle
1762
+ cycle.add_function(function)
1763
+
1764
+ profile[TIME] = profile[TIME] + function[TIME]
1765
+
1766
+ for cycle in compat_itervalues(cycles):
1767
+ profile.add_cycle(cycle)
1768
+
1769
+ # Compute derived events.
1770
+ profile.validate()
1771
+ profile.ratio(TIME_RATIO, TIME)
1772
+ # Lacking call counts, fake call ratios based on total times.
1773
+ profile.call_ratios(TOTAL_TIME_RATIO)
1774
+ # The TOTAL_TIME_RATIO of functions is already set. Propagate that
1775
+ # total time to the calls. (TOTAL_TIME is neither set nor used.)
1776
+ for function in compat_itervalues(profile.functions):
1777
+ for call in compat_itervalues(function.calls):
1778
+ if call.ratio is not None:
1779
+ callee = profile.functions[call.callee_id]
1780
+ call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]
1781
+
1782
+ return profile
1783
+
1784
+
1785
+ class CallgrindParser(LineParser):
1786
+ """Parser for valgrind's callgrind tool.
1787
+
1788
+ See also:
1789
+ - http://valgrind.org/docs/manual/cl-format.html
1790
+ """
1791
+
1792
+ _call_re = re.compile(r"^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$")
1793
+
1794
+ def __init__(self, infile):
1795
+ LineParser.__init__(self, infile)
1796
+
1797
+ # Textual positions
1798
+ self.position_ids = {}
1799
+ self.positions = {}
1800
+
1801
+ # Numeric positions
1802
+ self.num_positions = 1
1803
+ self.cost_positions = ["line"]
1804
+ self.last_positions = [0]
1805
+
1806
+ # Events
1807
+ self.num_events = 0
1808
+ self.cost_events = []
1809
+
1810
+ self.profile = Profile()
1811
+ self.profile[SAMPLES] = 0
1812
+
1813
+ def parse(self):
1814
+ # read lookahead
1815
+ self.readline()
1816
+
1817
+ self.parse_key("version")
1818
+ self.parse_key("creator")
1819
+ while self.parse_part():
1820
+ pass
1821
+ if not self.eof():
1822
+ sys.stderr.write("warning: line %u: unexpected line\n" % self.line_no)
1823
+ sys.stderr.write("%s\n" % self.lookahead())
1824
+
1825
+ # compute derived data
1826
+ self.profile.validate()
1827
+ self.profile.find_cycles()
1828
+ self.profile.ratio(TIME_RATIO, SAMPLES)
1829
+ self.profile.call_ratios(SAMPLES2)
1830
+ self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
1831
+
1832
+ return self.profile
1833
+
1834
+ def parse_part(self):
1835
+ if not self.parse_header_line():
1836
+ return False
1837
+ while self.parse_header_line():
1838
+ pass
1839
+ if not self.parse_body_line():
1840
+ return False
1841
+ while self.parse_body_line():
1842
+ pass
1843
+ return True
1844
+
1845
+ def parse_header_line(self):
1846
+ return (
1847
+ self.parse_empty()
1848
+ or self.parse_comment()
1849
+ or self.parse_part_detail()
1850
+ or self.parse_description()
1851
+ or self.parse_event_specification()
1852
+ or self.parse_cost_line_def()
1853
+ or self.parse_cost_summary()
1854
+ )
1855
+
1856
+ _detail_keys = set(("cmd", "pid", "thread", "part"))
1857
+
1858
+ def parse_part_detail(self):
1859
+ return self.parse_keys(self._detail_keys)
1860
+
1861
+ def parse_description(self):
1862
+ return self.parse_key("desc") is not None
1863
+
1864
+ def parse_event_specification(self):
1865
+ event = self.parse_key("event")
1866
+ if event is None:
1867
+ return False
1868
+ return True
1869
+
1870
+ def parse_cost_line_def(self):
1871
+ pair = self.parse_keys(("events", "positions"))
1872
+ if pair is None:
1873
+ return False
1874
+ key, value = pair
1875
+ items = value.split()
1876
+ if key == "events":
1877
+ self.num_events = len(items)
1878
+ self.cost_events = items
1879
+ if key == "positions":
1880
+ self.num_positions = len(items)
1881
+ self.cost_positions = items
1882
+ self.last_positions = [0] * self.num_positions
1883
+ return True
1884
+
1885
+ def parse_cost_summary(self):
1886
+ pair = self.parse_keys(("summary", "totals"))
1887
+ if pair is None:
1888
+ return False
1889
+ return True
1890
+
1891
+ def parse_body_line(self):
1892
+ return (
1893
+ self.parse_empty()
1894
+ or self.parse_comment()
1895
+ or self.parse_cost_line()
1896
+ or self.parse_position_spec()
1897
+ or self.parse_association_spec()
1898
+ )
1899
+
1900
+ __subpos_re = r"(0x[0-9a-fA-F]+|\d+|\+\d+|-\d+|\*)"
1901
+ _cost_re = re.compile(
1902
+ r"^" + __subpos_re + r"( +" + __subpos_re + r")*" + r"( +\d+)*" + "$"
1903
+ )
1904
+
1905
+ def parse_cost_line(self, calls=None):
1906
+ line = self.lookahead().rstrip()
1907
+ mo = self._cost_re.match(line)
1908
+ if not mo:
1909
+ return False
1910
+
1911
+ function = self.get_function()
1912
+
1913
+ if calls is None:
1914
+ # Unlike other aspects, call object (cob) is relative not to the
1915
+ # last call object, but to the caller's object (ob), so try to
1916
+ # update it when processing a functions cost line
1917
+ try:
1918
+ self.positions["cob"] = self.positions["ob"]
1919
+ except KeyError:
1920
+ pass
1921
+
1922
+ values = line.split()
1923
+ assert len(values) <= self.num_positions + self.num_events
1924
+
1925
+ positions = values[0 : self.num_positions]
1926
+ events = values[self.num_positions :]
1927
+ events += ["0"] * (self.num_events - len(events))
1928
+
1929
+ for i in range(self.num_positions):
1930
+ position = positions[i]
1931
+ if position == "*":
1932
+ position = self.last_positions[i]
1933
+ elif position[0] in "-+":
1934
+ position = self.last_positions[i] + int(position)
1935
+ elif position.startswith("0x"):
1936
+ position = int(position, 16)
1937
+ else:
1938
+ position = int(position)
1939
+ self.last_positions[i] = position
1940
+
1941
+ events = [float(event) for event in events]
1942
+
1943
+ if calls is None:
1944
+ function[SAMPLES] += events[0]
1945
+ self.profile[SAMPLES] += events[0]
1946
+ else:
1947
+ callee = self.get_callee()
1948
+ callee.called += calls
1949
+
1950
+ try:
1951
+ call = function.calls[callee.id]
1952
+ except KeyError:
1953
+ call = Call(callee.id)
1954
+ call[CALLS] = calls
1955
+ call[SAMPLES2] = events[0]
1956
+ function.add_call(call)
1957
+ else:
1958
+ call[CALLS] += calls
1959
+ call[SAMPLES2] += events[0]
1960
+
1961
+ self.consume()
1962
+ return True
1963
+
1964
+ def parse_association_spec(self):
1965
+ line = self.lookahead()
1966
+ if not line.startswith("calls="):
1967
+ return False
1968
+
1969
+ _, values = line.split("=", 1)
1970
+ values = values.strip().split()
1971
+ calls = int(values[0])
1972
+ call_position = values[1:]
1973
+ self.consume()
1974
+
1975
+ self.parse_cost_line(calls)
1976
+
1977
+ return True
1978
+
1979
+ _position_re = re.compile(
1980
+ r"^(?P<position>[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P<id>\d+)\))?(?:\s*(?P<name>.+))?"
1981
+ )
1982
+
1983
+ _position_table_map = {
1984
+ "ob": "ob",
1985
+ "fl": "fl",
1986
+ "fi": "fl",
1987
+ "fe": "fl",
1988
+ "fn": "fn",
1989
+ "cob": "ob",
1990
+ "cfl": "fl",
1991
+ "cfi": "fl",
1992
+ "cfe": "fl",
1993
+ "cfn": "fn",
1994
+ "jfi": "fl",
1995
+ }
1996
+
1997
+ _position_map = {
1998
+ "ob": "ob",
1999
+ "fl": "fl",
2000
+ "fi": "fl",
2001
+ "fe": "fl",
2002
+ "fn": "fn",
2003
+ "cob": "cob",
2004
+ "cfl": "cfl",
2005
+ "cfi": "cfl",
2006
+ "cfe": "cfl",
2007
+ "cfn": "cfn",
2008
+ "jfi": "jfi",
2009
+ }
2010
+
2011
+ def parse_position_spec(self):
2012
+ line = self.lookahead()
2013
+
2014
+ if line.startswith("jump=") or line.startswith("jcnd="):
2015
+ self.consume()
2016
+ return True
2017
+
2018
+ mo = self._position_re.match(line)
2019
+ if not mo:
2020
+ return False
2021
+
2022
+ position, id, name = mo.groups()
2023
+ if id:
2024
+ table = self._position_table_map[position]
2025
+ if name:
2026
+ self.position_ids[(table, id)] = name
2027
+ else:
2028
+ name = self.position_ids.get((table, id), "")
2029
+ self.positions[self._position_map[position]] = name
2030
+
2031
+ self.consume()
2032
+ return True
2033
+
2034
+ def parse_empty(self):
2035
+ if self.eof():
2036
+ return False
2037
+ line = self.lookahead()
2038
+ if line.strip():
2039
+ return False
2040
+ self.consume()
2041
+ return True
2042
+
2043
+ def parse_comment(self):
2044
+ line = self.lookahead()
2045
+ if not line.startswith("#"):
2046
+ return False
2047
+ self.consume()
2048
+ return True
2049
+
2050
+ _key_re = re.compile(r"^(\w+):")
2051
+
2052
+ def parse_key(self, key):
2053
+ pair = self.parse_keys((key,))
2054
+ if not pair:
2055
+ return None
2056
+ key, value = pair
2057
+ return value
2058
+
2059
+ def parse_keys(self, keys):
2060
+ line = self.lookahead()
2061
+ mo = self._key_re.match(line)
2062
+ if not mo:
2063
+ return None
2064
+ key, value = line.split(":", 1)
2065
+ if key not in keys:
2066
+ return None
2067
+ value = value.strip()
2068
+ self.consume()
2069
+ return key, value
2070
+
2071
+ def make_function(self, module, filename, name):
2072
+ # FIXME: module and filename are not being tracked reliably
2073
+ # id = '|'.join((module, filename, name))
2074
+ id = name
2075
+ try:
2076
+ function = self.profile.functions[id]
2077
+ except KeyError:
2078
+ function = Function(id, name)
2079
+ if module:
2080
+ function.module = os.path.basename(module)
2081
+ function[SAMPLES] = 0
2082
+ function.called = 0
2083
+ self.profile.add_function(function)
2084
+ return function
2085
+
2086
+ def get_function(self):
2087
+ module = self.positions.get("ob", "")
2088
+ filename = self.positions.get("fl", "")
2089
+ function = self.positions.get("fn", "")
2090
+ return self.make_function(module, filename, function)
2091
+
2092
+ def get_callee(self):
2093
+ module = self.positions.get("cob", "")
2094
+ filename = self.positions.get("cfi", "")
2095
+ function = self.positions.get("cfn", "")
2096
+ return self.make_function(module, filename, function)
2097
+
2098
+ def readline(self):
2099
+ # Override LineParser.readline to ignore comment lines
2100
+ while True:
2101
+ LineParser.readline(self)
2102
+ if self.eof() or not self.lookahead().startswith("#"):
2103
+ break
2104
+
2105
+
2106
+ class PerfParser(LineParser):
2107
+ """Parser for linux perf callgraph output.
2108
+
2109
+ It expects output generated with
2110
+
2111
+ perf record -g
2112
+ perf script | gprof2dot.py --format=perf
2113
+ """
2114
+
2115
+ def __init__(self, infile):
2116
+ LineParser.__init__(self, infile)
2117
+ self.profile = Profile()
2118
+
2119
+ def readline(self):
2120
+ # Override LineParser.readline to ignore comment lines
2121
+ while True:
2122
+ LineParser.readline(self)
2123
+ if self.eof() or not self.lookahead().startswith("#"):
2124
+ break
2125
+
2126
+ def parse(self):
2127
+ # read lookahead
2128
+ self.readline()
2129
+
2130
+ profile = self.profile
2131
+ profile[SAMPLES] = 0
2132
+ while not self.eof():
2133
+ self.parse_event()
2134
+
2135
+ # compute derived data
2136
+ profile.validate()
2137
+ profile.find_cycles()
2138
+ profile.ratio(TIME_RATIO, SAMPLES)
2139
+ profile.call_ratios(SAMPLES2)
2140
+ if totalMethod == "callratios":
2141
+ # Heuristic approach. TOTAL_SAMPLES is unused.
2142
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2143
+ elif totalMethod == "callstacks":
2144
+ # Use the actual call chains for functions.
2145
+ profile[TOTAL_SAMPLES] = profile[SAMPLES]
2146
+ profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)
2147
+ # Then propagate that total time to the calls.
2148
+ for function in compat_itervalues(profile.functions):
2149
+ for call in compat_itervalues(function.calls):
2150
+ if call.ratio is not None:
2151
+ callee = profile.functions[call.callee_id]
2152
+ call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]
2153
+ else:
2154
+ assert False
2155
+
2156
+ return profile
2157
+
2158
+ def parse_event(self):
2159
+ if self.eof():
2160
+ return
2161
+
2162
+ line = self.consume()
2163
+ assert line
2164
+
2165
+ callchain = self.parse_callchain()
2166
+ if not callchain:
2167
+ return
2168
+
2169
+ callee = callchain[0]
2170
+ callee[SAMPLES] += 1
2171
+ self.profile[SAMPLES] += 1
2172
+
2173
+ for caller in callchain[1:]:
2174
+ try:
2175
+ call = caller.calls[callee.id]
2176
+ except KeyError:
2177
+ call = Call(callee.id)
2178
+ call[SAMPLES2] = 1
2179
+ caller.add_call(call)
2180
+ else:
2181
+ call[SAMPLES2] += 1
2182
+
2183
+ callee = caller
2184
+
2185
+ # Increment TOTAL_SAMPLES only once on each function.
2186
+ stack = set(callchain)
2187
+ for function in stack:
2188
+ function[TOTAL_SAMPLES] += 1
2189
+
2190
+ def parse_callchain(self):
2191
+ callchain = []
2192
+ while self.lookahead():
2193
+ function = self.parse_call()
2194
+ if function is None:
2195
+ break
2196
+ callchain.append(function)
2197
+ if self.lookahead() == "":
2198
+ self.consume()
2199
+ return callchain
2200
+
2201
+ call_re = re.compile(
2202
+ r"^\s+(?P<address>[0-9a-fA-F]+)\s+(?P<symbol>.*)\s+\((?P<module>.*)\)$"
2203
+ )
2204
+ addr2_re = re.compile(r"\+0x[0-9a-fA-F]+$")
2205
+
2206
+ def parse_call(self):
2207
+ line = self.consume()
2208
+ mo = self.call_re.match(line)
2209
+ assert mo
2210
+ if not mo:
2211
+ return None
2212
+
2213
+ function_name = mo.group("symbol")
2214
+
2215
+ # If present, amputate program counter from function name.
2216
+ if function_name:
2217
+ function_name = re.sub(self.addr2_re, "", function_name)
2218
+
2219
+ if not function_name or function_name == "[unknown]":
2220
+ function_name = mo.group("address")
2221
+
2222
+ module = mo.group("module")
2223
+
2224
+ function_id = function_name + ":" + module
2225
+
2226
+ try:
2227
+ function = self.profile.functions[function_id]
2228
+ except KeyError:
2229
+ function = Function(function_id, function_name)
2230
+ function.module = os.path.basename(module)
2231
+ function[SAMPLES] = 0
2232
+ function[TOTAL_SAMPLES] = 0
2233
+ self.profile.add_function(function)
2234
+
2235
+ return function
2236
+
2237
+
2238
+ class OprofileParser(LineParser):
2239
+ """Parser for oprofile callgraph output.
2240
+
2241
+ See also:
2242
+ - http://oprofile.sourceforge.net/doc/opreport.html#opreport-callgraph
2243
+ """
2244
+
2245
+ _fields_re = {
2246
+ "samples": r"(\d+)",
2247
+ "%": r"(\S+)",
2248
+ "linenr info": r"(?P<source>\(no location information\)|\S+:\d+)",
2249
+ "image name": r"(?P<image>\S+(?:\s\(tgid:[^)]*\))?)",
2250
+ "app name": r"(?P<application>\S+)",
2251
+ "symbol name": r"(?P<symbol>\(no symbols\)|.+?)",
2252
+ }
2253
+
2254
+ def __init__(self, infile):
2255
+ LineParser.__init__(self, infile)
2256
+ self.entries = {}
2257
+ self.entry_re = None
2258
+
2259
+ def add_entry(self, callers, function, callees):
2260
+ try:
2261
+ entry = self.entries[function.id]
2262
+ except KeyError:
2263
+ self.entries[function.id] = (callers, function, callees)
2264
+ else:
2265
+ callers_total, function_total, callees_total = entry
2266
+ self.update_subentries_dict(callers_total, callers)
2267
+ function_total.samples += function.samples
2268
+ self.update_subentries_dict(callees_total, callees)
2269
+
2270
+ def update_subentries_dict(self, totals, partials):
2271
+ for partial in compat_itervalues(partials):
2272
+ try:
2273
+ total = totals[partial.id]
2274
+ except KeyError:
2275
+ totals[partial.id] = partial
2276
+ else:
2277
+ total.samples += partial.samples
2278
+
2279
+ def parse(self):
2280
+ # read lookahead
2281
+ self.readline()
2282
+
2283
+ self.parse_header()
2284
+ while self.lookahead():
2285
+ self.parse_entry()
2286
+
2287
+ profile = Profile()
2288
+
2289
+ reverse_call_samples = {}
2290
+
2291
+ # populate the profile
2292
+ profile[SAMPLES] = 0
2293
+ for _callers, _function, _callees in compat_itervalues(self.entries):
2294
+ function = Function(_function.id, _function.name)
2295
+ function[SAMPLES] = _function.samples
2296
+ profile.add_function(function)
2297
+ profile[SAMPLES] += _function.samples
2298
+
2299
+ if _function.application:
2300
+ function.process = os.path.basename(_function.application)
2301
+ if _function.image:
2302
+ function.module = os.path.basename(_function.image)
2303
+
2304
+ total_callee_samples = 0
2305
+ for _callee in compat_itervalues(_callees):
2306
+ total_callee_samples += _callee.samples
2307
+
2308
+ for _callee in compat_itervalues(_callees):
2309
+ if not _callee.self:
2310
+ call = Call(_callee.id)
2311
+ call[SAMPLES2] = _callee.samples
2312
+ function.add_call(call)
2313
+
2314
+ # compute derived data
2315
+ profile.validate()
2316
+ profile.find_cycles()
2317
+ profile.ratio(TIME_RATIO, SAMPLES)
2318
+ profile.call_ratios(SAMPLES2)
2319
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2320
+
2321
+ return profile
2322
+
2323
+ def parse_header(self):
2324
+ while not self.match_header():
2325
+ self.consume()
2326
+ line = self.lookahead()
2327
+ fields = re.split(r"\s\s+", line)
2328
+ entry_re = (
2329
+ r"^\s*"
2330
+ + r"\s+".join([self._fields_re[field] for field in fields])
2331
+ + r"(?P<self>\s+\[self\])?$"
2332
+ )
2333
+ self.entry_re = re.compile(entry_re)
2334
+ self.skip_separator()
2335
+
2336
+ def parse_entry(self):
2337
+ callers = self.parse_subentries()
2338
+ if self.match_primary():
2339
+ function = self.parse_subentry()
2340
+ if function is not None:
2341
+ callees = self.parse_subentries()
2342
+ self.add_entry(callers, function, callees)
2343
+ self.skip_separator()
2344
+
2345
+ def parse_subentries(self):
2346
+ subentries = {}
2347
+ while self.match_secondary():
2348
+ subentry = self.parse_subentry()
2349
+ subentries[subentry.id] = subentry
2350
+ return subentries
2351
+
2352
+ def parse_subentry(self):
2353
+ entry = Struct()
2354
+ line = self.consume()
2355
+ mo = self.entry_re.match(line)
2356
+ if not mo:
2357
+ raise ParseError("failed to parse", line)
2358
+ fields = mo.groupdict()
2359
+ entry.samples = int(mo.group(1))
2360
+ if "source" in fields and fields["source"] != "(no location information)":
2361
+ source = fields["source"]
2362
+ filename, lineno = source.split(":")
2363
+ entry.filename = filename
2364
+ entry.lineno = int(lineno)
2365
+ else:
2366
+ source = ""
2367
+ entry.filename = None
2368
+ entry.lineno = None
2369
+ entry.image = fields.get("image", "")
2370
+ entry.application = fields.get("application", "")
2371
+ if "symbol" in fields and fields["symbol"] != "(no symbols)":
2372
+ entry.symbol = fields["symbol"]
2373
+ else:
2374
+ entry.symbol = ""
2375
+ if entry.symbol.startswith('"') and entry.symbol.endswith('"'):
2376
+ entry.symbol = entry.symbol[1:-1]
2377
+ entry.id = ":".join((entry.application, entry.image, source, entry.symbol))
2378
+ entry.self = fields.get("self", None) != None
2379
+ if entry.self:
2380
+ entry.id += ":self"
2381
+ if entry.symbol:
2382
+ entry.name = entry.symbol
2383
+ else:
2384
+ entry.name = entry.image
2385
+ return entry
2386
+
2387
+ def skip_separator(self):
2388
+ while not self.match_separator():
2389
+ self.consume()
2390
+ self.consume()
2391
+
2392
+ def match_header(self):
2393
+ line = self.lookahead()
2394
+ return line.startswith("samples")
2395
+
2396
+ def match_separator(self):
2397
+ line = self.lookahead()
2398
+ return line == "-" * len(line)
2399
+
2400
+ def match_primary(self):
2401
+ line = self.lookahead()
2402
+ return not line[:1].isspace()
2403
+
2404
+ def match_secondary(self):
2405
+ line = self.lookahead()
2406
+ return line[:1].isspace()
2407
+
2408
+
2409
+ class HProfParser(LineParser):
2410
+ """Parser for java hprof output
2411
+
2412
+ See also:
2413
+ - http://java.sun.com/developer/technicalArticles/Programming/HPROF.html
2414
+ """
2415
+
2416
+ trace_re = re.compile(r"\t(.*)\((.*):(.*)\)")
2417
+ trace_id_re = re.compile(r"^TRACE (\d+):$")
2418
+
2419
+ def __init__(self, infile):
2420
+ LineParser.__init__(self, infile)
2421
+ self.traces = {}
2422
+ self.samples = {}
2423
+
2424
+ def parse(self):
2425
+ # read lookahead
2426
+ self.readline()
2427
+
2428
+ while not self.lookahead().startswith("------"):
2429
+ self.consume()
2430
+ while not self.lookahead().startswith("TRACE "):
2431
+ self.consume()
2432
+
2433
+ self.parse_traces()
2434
+
2435
+ while not self.lookahead().startswith("CPU"):
2436
+ self.consume()
2437
+
2438
+ self.parse_samples()
2439
+
2440
+ # populate the profile
2441
+ profile = Profile()
2442
+ profile[SAMPLES] = 0
2443
+
2444
+ functions = {}
2445
+
2446
+ # build up callgraph
2447
+ for id, trace in compat_iteritems(self.traces):
2448
+ if not id in self.samples:
2449
+ continue
2450
+ mtime = self.samples[id][0]
2451
+ last = None
2452
+
2453
+ for func, file, line in trace:
2454
+ if not func in functions:
2455
+ function = Function(func, func)
2456
+ function[SAMPLES] = 0
2457
+ profile.add_function(function)
2458
+ functions[func] = function
2459
+
2460
+ function = functions[func]
2461
+ # allocate time to the deepest method in the trace
2462
+ if not last:
2463
+ function[SAMPLES] += mtime
2464
+ profile[SAMPLES] += mtime
2465
+ else:
2466
+ c = function.get_call(last)
2467
+ c[SAMPLES2] += mtime
2468
+
2469
+ last = func
2470
+
2471
+ # compute derived data
2472
+ profile.validate()
2473
+ profile.find_cycles()
2474
+ profile.ratio(TIME_RATIO, SAMPLES)
2475
+ profile.call_ratios(SAMPLES2)
2476
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2477
+
2478
+ return profile
2479
+
2480
+ def parse_traces(self):
2481
+ while self.lookahead().startswith("TRACE "):
2482
+ self.parse_trace()
2483
+
2484
+ def parse_trace(self):
2485
+ l = self.consume()
2486
+ mo = self.trace_id_re.match(l)
2487
+ tid = mo.group(1)
2488
+ last = None
2489
+ trace = []
2490
+
2491
+ while self.lookahead().startswith("\t"):
2492
+ l = self.consume()
2493
+ match = self.trace_re.search(l)
2494
+ if not match:
2495
+ # sys.stderr.write('Invalid line: %s\n' % l)
2496
+ break
2497
+ else:
2498
+ function_name, file, line = match.groups()
2499
+ trace += [(function_name, file, line)]
2500
+
2501
+ self.traces[int(tid)] = trace
2502
+
2503
+ def parse_samples(self):
2504
+ self.consume()
2505
+ self.consume()
2506
+
2507
+ while not self.lookahead().startswith("CPU"):
2508
+ (
2509
+ rank,
2510
+ percent_self,
2511
+ percent_accum,
2512
+ count,
2513
+ traceid,
2514
+ method,
2515
+ ) = self.lookahead().split()
2516
+ self.samples[int(traceid)] = (int(count), method)
2517
+ self.consume()
2518
+
2519
+
2520
+ class SysprofParser(XmlParser):
2521
+ def __init__(self, stream):
2522
+ XmlParser.__init__(self, stream)
2523
+
2524
+ def parse(self):
2525
+ objects = {}
2526
+ nodes = {}
2527
+
2528
+ self.element_start("profile")
2529
+ while self.token.type == XML_ELEMENT_START:
2530
+ if self.token.name_or_data == "objects":
2531
+ assert not objects
2532
+ objects = self.parse_items("objects")
2533
+ elif self.token.name_or_data == "nodes":
2534
+ assert not nodes
2535
+ nodes = self.parse_items("nodes")
2536
+ else:
2537
+ self.parse_value(self.token.name_or_data)
2538
+ self.element_end("profile")
2539
+
2540
+ return self.build_profile(objects, nodes)
2541
+
2542
+ def parse_items(self, name):
2543
+ assert name[-1] == "s"
2544
+ items = {}
2545
+ self.element_start(name)
2546
+ while self.token.type == XML_ELEMENT_START:
2547
+ id, values = self.parse_item(name[:-1])
2548
+ assert id not in items
2549
+ items[id] = values
2550
+ self.element_end(name)
2551
+ return items
2552
+
2553
+ def parse_item(self, name):
2554
+ attrs = self.element_start(name)
2555
+ id = int(attrs["id"])
2556
+ values = self.parse_values()
2557
+ self.element_end(name)
2558
+ return id, values
2559
+
2560
+ def parse_values(self):
2561
+ values = {}
2562
+ while self.token.type == XML_ELEMENT_START:
2563
+ name = self.token.name_or_data
2564
+ value = self.parse_value(name)
2565
+ assert name not in values
2566
+ values[name] = value
2567
+ return values
2568
+
2569
+ def parse_value(self, tag):
2570
+ self.element_start(tag)
2571
+ value = self.character_data()
2572
+ self.element_end(tag)
2573
+ if value.isdigit():
2574
+ return int(value)
2575
+ if value.startswith('"') and value.endswith('"'):
2576
+ return value[1:-1]
2577
+ return value
2578
+
2579
+ def build_profile(self, objects, nodes):
2580
+ profile = Profile()
2581
+
2582
+ profile[SAMPLES] = 0
2583
+ for id, object in compat_iteritems(objects):
2584
+ # Ignore fake objects (process names, modules, "Everything", "kernel", etc.)
2585
+ if object["self"] == 0:
2586
+ continue
2587
+
2588
+ function = Function(id, object["name"])
2589
+ function[SAMPLES] = object["self"]
2590
+ profile.add_function(function)
2591
+ profile[SAMPLES] += function[SAMPLES]
2592
+
2593
+ for id, node in compat_iteritems(nodes):
2594
+ # Ignore fake calls
2595
+ if node["self"] == 0:
2596
+ continue
2597
+
2598
+ # Find a non-ignored parent
2599
+ parent_id = node["parent"]
2600
+ while parent_id != 0:
2601
+ parent = nodes[parent_id]
2602
+ caller_id = parent["object"]
2603
+ if objects[caller_id]["self"] != 0:
2604
+ break
2605
+ parent_id = parent["parent"]
2606
+ if parent_id == 0:
2607
+ continue
2608
+
2609
+ callee_id = node["object"]
2610
+
2611
+ assert objects[caller_id]["self"]
2612
+ assert objects[callee_id]["self"]
2613
+
2614
+ function = profile.functions[caller_id]
2615
+
2616
+ samples = node["self"]
2617
+ try:
2618
+ call = function.calls[callee_id]
2619
+ except KeyError:
2620
+ call = Call(callee_id)
2621
+ call[SAMPLES2] = samples
2622
+ function.add_call(call)
2623
+ else:
2624
+ call[SAMPLES2] += samples
2625
+
2626
+ # Compute derived events
2627
+ profile.validate()
2628
+ profile.find_cycles()
2629
+ profile.ratio(TIME_RATIO, SAMPLES)
2630
+ profile.call_ratios(SAMPLES2)
2631
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2632
+
2633
+ return profile
2634
+
2635
+
2636
+ class XPerfParser(Parser):
2637
+ """Parser for CSVs generated by XPerf, from Microsoft Windows Performance Tools."""
2638
+
2639
+ def __init__(self, stream):
2640
+ Parser.__init__(self)
2641
+ self.stream = stream
2642
+ self.profile = Profile()
2643
+ self.profile[SAMPLES] = 0
2644
+ self.column = {}
2645
+
2646
+ def parse(self):
2647
+ import csv
2648
+
2649
+ reader = csv.reader(
2650
+ self.stream,
2651
+ delimiter=",",
2652
+ quotechar=None,
2653
+ escapechar=None,
2654
+ doublequote=False,
2655
+ skipinitialspace=True,
2656
+ lineterminator="\r\n",
2657
+ quoting=csv.QUOTE_NONE,
2658
+ )
2659
+ header = True
2660
+ for row in reader:
2661
+ if header:
2662
+ self.parse_header(row)
2663
+ header = False
2664
+ else:
2665
+ self.parse_row(row)
2666
+
2667
+ # compute derived data
2668
+ self.profile.validate()
2669
+ self.profile.find_cycles()
2670
+ self.profile.ratio(TIME_RATIO, SAMPLES)
2671
+ self.profile.call_ratios(SAMPLES2)
2672
+ self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2673
+
2674
+ return self.profile
2675
+
2676
+ def parse_header(self, row):
2677
+ for column in range(len(row)):
2678
+ name = row[column]
2679
+ assert name not in self.column
2680
+ self.column[name] = column
2681
+
2682
+ def parse_row(self, row):
2683
+ fields = {}
2684
+ for name, column in compat_iteritems(self.column):
2685
+ value = row[column]
2686
+ for factory in int, float:
2687
+ try:
2688
+ value = factory(value)
2689
+ except ValueError:
2690
+ pass
2691
+ else:
2692
+ break
2693
+ fields[name] = value
2694
+
2695
+ process = fields["Process Name"]
2696
+ symbol = fields["Module"] + "!" + fields["Function"]
2697
+ weight = fields["Weight"]
2698
+ count = fields["Count"]
2699
+
2700
+ if process == "Idle":
2701
+ return
2702
+
2703
+ function = self.get_function(process, symbol)
2704
+ function[SAMPLES] += weight * count
2705
+ self.profile[SAMPLES] += weight * count
2706
+
2707
+ stack = fields["Stack"]
2708
+ if stack != "?":
2709
+ stack = stack.split("/")
2710
+ assert stack[0] == "[Root]"
2711
+ if stack[-1] != symbol:
2712
+ # XXX: some cases the sampled function does not appear in the stack
2713
+ stack.append(symbol)
2714
+ caller = None
2715
+ for symbol in stack[1:]:
2716
+ callee = self.get_function(process, symbol)
2717
+ if caller is not None:
2718
+ try:
2719
+ call = caller.calls[callee.id]
2720
+ except KeyError:
2721
+ call = Call(callee.id)
2722
+ call[SAMPLES2] = count
2723
+ caller.add_call(call)
2724
+ else:
2725
+ call[SAMPLES2] += count
2726
+ caller = callee
2727
+
2728
+ def get_function(self, process, symbol):
2729
+ function_id = process + "!" + symbol
2730
+
2731
+ try:
2732
+ function = self.profile.functions[function_id]
2733
+ except KeyError:
2734
+ module, name = symbol.split("!", 1)
2735
+ function = Function(function_id, name)
2736
+ function.process = process
2737
+ function.module = module
2738
+ function[SAMPLES] = 0
2739
+ self.profile.add_function(function)
2740
+
2741
+ return function
2742
+
2743
+
2744
+ class SleepyParser(Parser):
2745
+ """Parser for GNU gprof output.
2746
+
2747
+ See also:
2748
+ - http://www.codersnotes.com/sleepy/
2749
+ - http://sleepygraph.sourceforge.net/
2750
+ """
2751
+
2752
+ stdinInput = False
2753
+
2754
+ def __init__(self, filename):
2755
+ Parser.__init__(self)
2756
+
2757
+ from zipfile import ZipFile
2758
+
2759
+ self.database = ZipFile(filename)
2760
+
2761
+ self.symbols = {}
2762
+ self.calls = {}
2763
+
2764
+ self.profile = Profile()
2765
+
2766
+ _symbol_re = re.compile(
2767
+ r"^(?P<id>\w+)"
2768
+ + r'\s+"(?P<module>[^"]*)"'
2769
+ + r'\s+"(?P<procname>[^"]*)"'
2770
+ + r'\s+"(?P<sourcefile>[^"]*)"'
2771
+ + r"\s+(?P<sourceline>\d+)$"
2772
+ )
2773
+
2774
+ def openEntry(self, name):
2775
+ # Some versions of verysleepy use lowercase filenames
2776
+ for database_name in self.database.namelist():
2777
+ if name.lower() == database_name.lower():
2778
+ name = database_name
2779
+ break
2780
+
2781
+ return self.database.open(name, "r")
2782
+
2783
+ def parse_symbols(self):
2784
+ for line in self.openEntry("Symbols.txt"):
2785
+ line = line.decode("UTF-8").rstrip("\r\n")
2786
+
2787
+ mo = self._symbol_re.match(line)
2788
+ if mo:
2789
+ symbol_id, module, procname, sourcefile, sourceline = mo.groups()
2790
+
2791
+ function_id = ":".join([module, procname])
2792
+
2793
+ try:
2794
+ function = self.profile.functions[function_id]
2795
+ except KeyError:
2796
+ function = Function(function_id, procname)
2797
+ function.module = module
2798
+ function[SAMPLES] = 0
2799
+ self.profile.add_function(function)
2800
+
2801
+ self.symbols[symbol_id] = function
2802
+
2803
+ def parse_callstacks(self):
2804
+ for line in self.openEntry("Callstacks.txt"):
2805
+ line = line.decode("UTF-8").rstrip("\r\n")
2806
+
2807
+ fields = line.split()
2808
+ samples = float(fields[0])
2809
+ callstack = fields[1:]
2810
+
2811
+ callstack = [self.symbols[symbol_id] for symbol_id in callstack]
2812
+
2813
+ callee = callstack[0]
2814
+
2815
+ callee[SAMPLES] += samples
2816
+ self.profile[SAMPLES] += samples
2817
+
2818
+ for caller in callstack[1:]:
2819
+ try:
2820
+ call = caller.calls[callee.id]
2821
+ except KeyError:
2822
+ call = Call(callee.id)
2823
+ call[SAMPLES2] = samples
2824
+ caller.add_call(call)
2825
+ else:
2826
+ call[SAMPLES2] += samples
2827
+
2828
+ callee = caller
2829
+
2830
+ def parse(self):
2831
+ profile = self.profile
2832
+ profile[SAMPLES] = 0
2833
+
2834
+ self.parse_symbols()
2835
+ self.parse_callstacks()
2836
+
2837
+ # Compute derived events
2838
+ profile.validate()
2839
+ profile.find_cycles()
2840
+ profile.ratio(TIME_RATIO, SAMPLES)
2841
+ profile.call_ratios(SAMPLES2)
2842
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2843
+
2844
+ return profile
2845
+
2846
+
2847
+ class PstatsParser:
2848
+ """Parser python profiling statistics saved with te pstats module."""
2849
+
2850
+ stdinInput = False
2851
+ multipleInput = True
2852
+
2853
+ def __init__(self, *filename):
2854
+ import pstats
2855
+
2856
+ try:
2857
+ self.stats = pstats.Stats(*filename)
2858
+ except ValueError:
2859
+ if PYTHON_3:
2860
+ sys.stderr.write(
2861
+ "error: failed to load %s, maybe they are generated by different python version?\n"
2862
+ % ", ".join(filename)
2863
+ )
2864
+ sys.exit(1)
2865
+ import hotshot.stats
2866
+
2867
+ self.stats = hotshot.stats.load(filename[0])
2868
+ self.profile = Profile()
2869
+ self.function_ids = {}
2870
+
2871
+ def get_function_name(self, key):
2872
+ filename, line, name = key
2873
+ module = os.path.splitext(filename)[0]
2874
+ module = os.path.basename(module)
2875
+ return "%s:%d:%s" % (module, line, name)
2876
+
2877
+ def get_function(self, key):
2878
+ try:
2879
+ id = self.function_ids[key]
2880
+ except KeyError:
2881
+ id = len(self.function_ids)
2882
+ name = self.get_function_name(key)
2883
+ function = Function(id, name)
2884
+ function.filename = key[0]
2885
+ self.profile.functions[id] = function
2886
+ self.function_ids[key] = id
2887
+ else:
2888
+ function = self.profile.functions[id]
2889
+ return function
2890
+
2891
+ def parse(self):
2892
+ self.profile[TIME] = 0.0
2893
+ self.profile[TOTAL_TIME] = self.stats.total_tt
2894
+ for fn, (cc, nc, tt, ct, callers) in compat_iteritems(self.stats.stats):
2895
+ callee = self.get_function(fn)
2896
+ callee.called = nc
2897
+ callee[TOTAL_TIME] = ct
2898
+ callee[TIME] = tt
2899
+ self.profile[TIME] += tt
2900
+ self.profile[TOTAL_TIME] = max(self.profile[TOTAL_TIME], ct)
2901
+ for fn, value in compat_iteritems(callers):
2902
+ caller = self.get_function(fn)
2903
+ call = Call(callee.id)
2904
+ if isinstance(value, tuple):
2905
+ for i in xrange(0, len(value), 4):
2906
+ nc, cc, tt, ct = value[i : i + 4]
2907
+ if CALLS in call:
2908
+ call[CALLS] += cc
2909
+ else:
2910
+ call[CALLS] = cc
2911
+
2912
+ if TOTAL_TIME in call:
2913
+ call[TOTAL_TIME] += ct
2914
+ else:
2915
+ call[TOTAL_TIME] = ct
2916
+
2917
+ else:
2918
+ call[CALLS] = value
2919
+ call[TOTAL_TIME] = ratio(value, nc) * ct
2920
+
2921
+ caller.add_call(call)
2922
+
2923
+ if False:
2924
+ self.stats.print_stats()
2925
+ self.stats.print_callees()
2926
+
2927
+ # Compute derived events
2928
+ self.profile.validate()
2929
+ self.profile.ratio(TIME_RATIO, TIME)
2930
+ self.profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
2931
+
2932
+ return self.profile
2933
+
2934
+
2935
+ class DtraceParser(LineParser):
2936
+ """Parser for linux perf callgraph output.
2937
+
2938
+ It expects output generated with
2939
+
2940
+ # Refer to https://github.com/brendangregg/FlameGraph#dtrace
2941
+ # 60 seconds of user-level stacks, including time spent in-kernel, for PID 12345 at 97 Hertz
2942
+ sudo dtrace -x ustackframes=100 -n 'profile-97 /pid == 12345/ { @[ustack()] = count(); } tick-60s { exit(0); }' -o out.user_stacks
2943
+
2944
+ # The dtrace output
2945
+ gprof2dot.py -f dtrace out.user_stacks
2946
+
2947
+ # Notice: sometimes, the dtrace outputs format may be latin-1, and gprof2dot will fail to parse it.
2948
+ # To solve this problem, you should use iconv to convert to UTF-8 explicitly.
2949
+ # TODO: add an encoding flag to tell gprof2dot how to decode the profile file.
2950
+ iconv -f ISO-8859-1 -t UTF-8 out.user_stacks | gprof2dot.py -f dtrace
2951
+ """
2952
+
2953
+ def __init__(self, infile):
2954
+ LineParser.__init__(self, infile)
2955
+ self.profile = Profile()
2956
+
2957
+ def readline(self):
2958
+ # Override LineParser.readline to ignore comment lines
2959
+ while True:
2960
+ LineParser.readline(self)
2961
+ if self.eof():
2962
+ break
2963
+
2964
+ line = self.lookahead().strip()
2965
+ if line.startswith("CPU"):
2966
+ # The format likes:
2967
+ # CPU ID FUNCTION:NAME
2968
+ # 1 29684 :tick-60s
2969
+ # Skip next line
2970
+ LineParser.readline(self)
2971
+ elif not line == "":
2972
+ break
2973
+
2974
+ def parse(self):
2975
+ # read lookahead
2976
+ self.readline()
2977
+
2978
+ profile = self.profile
2979
+ profile[SAMPLES] = 0
2980
+ while not self.eof():
2981
+ self.parse_event()
2982
+
2983
+ # compute derived data
2984
+ profile.validate()
2985
+ profile.find_cycles()
2986
+ profile.ratio(TIME_RATIO, SAMPLES)
2987
+ profile.call_ratios(SAMPLES2)
2988
+ if totalMethod == "callratios":
2989
+ # Heuristic approach. TOTAL_SAMPLES is unused.
2990
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2991
+ elif totalMethod == "callstacks":
2992
+ # Use the actual call chains for functions.
2993
+ profile[TOTAL_SAMPLES] = profile[SAMPLES]
2994
+ profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)
2995
+ # Then propagate that total time to the calls.
2996
+ for function in compat_itervalues(profile.functions):
2997
+ for call in compat_itervalues(function.calls):
2998
+ if call.ratio is not None:
2999
+ callee = profile.functions[call.callee_id]
3000
+ call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]
3001
+ else:
3002
+ assert False
3003
+
3004
+ return profile
3005
+
3006
+ def parse_event(self):
3007
+ if self.eof():
3008
+ return
3009
+
3010
+ callchain, count = self.parse_callchain()
3011
+ if not callchain:
3012
+ return
3013
+
3014
+ callee = callchain[0]
3015
+ callee[SAMPLES] += count
3016
+ self.profile[SAMPLES] += count
3017
+
3018
+ for caller in callchain[1:]:
3019
+ try:
3020
+ call = caller.calls[callee.id]
3021
+ except KeyError:
3022
+ call = Call(callee.id)
3023
+ call[SAMPLES2] = count
3024
+ caller.add_call(call)
3025
+ else:
3026
+ call[SAMPLES2] += count
3027
+
3028
+ callee = caller
3029
+
3030
+ # Increment TOTAL_SAMPLES only once on each function.
3031
+ stack = set(callchain)
3032
+ for function in stack:
3033
+ function[TOTAL_SAMPLES] += count
3034
+
3035
+ def parse_callchain(self):
3036
+ callchain = []
3037
+ count = 0
3038
+ while self.lookahead():
3039
+ function, count = self.parse_call()
3040
+ if function is None:
3041
+ break
3042
+ callchain.append(function)
3043
+ return callchain, count
3044
+
3045
+ call_re = re.compile(r"^\s+(?P<module>.*)`(?P<symbol>.*)")
3046
+ addr2_re = re.compile(r"\+0x[0-9a-fA-F]+$")
3047
+
3048
+ def parse_call(self):
3049
+ line = self.consume()
3050
+ mo = self.call_re.match(line)
3051
+ if not mo:
3052
+ # The line must be the stack count
3053
+ return None, int(line.strip())
3054
+
3055
+ function_name = mo.group("symbol")
3056
+
3057
+ # If present, amputate program counter from function name.
3058
+ if function_name:
3059
+ function_name = re.sub(self.addr2_re, "", function_name)
3060
+
3061
+ # if not function_name or function_name == '[unknown]':
3062
+ # function_name = mo.group('address')
3063
+
3064
+ module = mo.group("module")
3065
+
3066
+ function_id = function_name + ":" + module
3067
+
3068
+ try:
3069
+ function = self.profile.functions[function_id]
3070
+ except KeyError:
3071
+ function = Function(function_id, function_name)
3072
+ function.module = os.path.basename(module)
3073
+ function[SAMPLES] = 0
3074
+ function[TOTAL_SAMPLES] = 0
3075
+ self.profile.add_function(function)
3076
+
3077
+ return function, None
3078
+
3079
+
3080
+ formats = {
3081
+ "axe": AXEParser,
3082
+ "callgrind": CallgrindParser,
3083
+ "hprof": HProfParser,
3084
+ "json": JsonParser,
3085
+ "oprofile": OprofileParser,
3086
+ "perf": PerfParser,
3087
+ "prof": GprofParser,
3088
+ "pstats": PstatsParser,
3089
+ "sleepy": SleepyParser,
3090
+ "sysprof": SysprofParser,
3091
+ "xperf": XPerfParser,
3092
+ "dtrace": DtraceParser,
3093
+ }
3094
+
3095
+
3096
+ ########################################################################
3097
+ # Output
3098
+
3099
+
3100
+ class Theme:
3101
+ def __init__(
3102
+ self,
3103
+ bgcolor=(0.0, 0.0, 1.0),
3104
+ mincolor=(0.0, 0.0, 0.0),
3105
+ maxcolor=(0.0, 0.0, 1.0),
3106
+ fontname="Arial",
3107
+ fontcolor="white",
3108
+ nodestyle="filled",
3109
+ minfontsize=10.0,
3110
+ maxfontsize=10.0,
3111
+ minpenwidth=0.5,
3112
+ maxpenwidth=4.0,
3113
+ gamma=2.2,
3114
+ skew=1.0,
3115
+ ):
3116
+ self.bgcolor = bgcolor
3117
+ self.mincolor = mincolor
3118
+ self.maxcolor = maxcolor
3119
+ self.fontname = fontname
3120
+ self.fontcolor = fontcolor
3121
+ self.nodestyle = nodestyle
3122
+ self.minfontsize = minfontsize
3123
+ self.maxfontsize = maxfontsize
3124
+ self.minpenwidth = minpenwidth
3125
+ self.maxpenwidth = maxpenwidth
3126
+ self.gamma = gamma
3127
+ self.skew = skew
3128
+
3129
+ def graph_bgcolor(self):
3130
+ return self.hsl_to_rgb(*self.bgcolor)
3131
+
3132
+ def graph_fontname(self):
3133
+ return self.fontname
3134
+
3135
+ def graph_fontcolor(self):
3136
+ return self.fontcolor
3137
+
3138
+ def graph_fontsize(self):
3139
+ return self.minfontsize
3140
+
3141
+ def node_bgcolor(self, weight):
3142
+ return self.color(weight)
3143
+
3144
+ def node_fgcolor(self, weight):
3145
+ if self.nodestyle == "filled":
3146
+ return self.graph_bgcolor()
3147
+ else:
3148
+ return self.color(weight)
3149
+
3150
+ def node_fontsize(self, weight):
3151
+ return self.fontsize(weight)
3152
+
3153
+ def node_style(self):
3154
+ return self.nodestyle
3155
+
3156
+ def edge_color(self, weight):
3157
+ return self.color(weight)
3158
+
3159
+ def edge_fontsize(self, weight):
3160
+ return self.fontsize(weight)
3161
+
3162
+ def edge_penwidth(self, weight):
3163
+ return max(weight * self.maxpenwidth, self.minpenwidth)
3164
+
3165
+ def edge_arrowsize(self, weight):
3166
+ return 0.5 * math.sqrt(self.edge_penwidth(weight))
3167
+
3168
+ def fontsize(self, weight):
3169
+ return max(weight**2 * self.maxfontsize, self.minfontsize)
3170
+
3171
+ def color(self, weight):
3172
+ weight = min(max(weight, 0.0), 1.0)
3173
+
3174
+ hmin, smin, lmin = self.mincolor
3175
+ hmax, smax, lmax = self.maxcolor
3176
+
3177
+ if self.skew < 0:
3178
+ raise ValueError("Skew must be greater than 0")
3179
+ elif self.skew == 1.0:
3180
+ h = hmin + weight * (hmax - hmin)
3181
+ s = smin + weight * (smax - smin)
3182
+ l = lmin + weight * (lmax - lmin)
3183
+ else:
3184
+ base = self.skew
3185
+ h = hmin + ((hmax - hmin) * (-1.0 + (base**weight)) / (base - 1.0))
3186
+ s = smin + ((smax - smin) * (-1.0 + (base**weight)) / (base - 1.0))
3187
+ l = lmin + ((lmax - lmin) * (-1.0 + (base**weight)) / (base - 1.0))
3188
+
3189
+ return self.hsl_to_rgb(h, s, l)
3190
+
3191
+ def hsl_to_rgb(self, h, s, l):
3192
+ """Convert a color from HSL color-model to RGB.
3193
+
3194
+ See also:
3195
+ - http://www.w3.org/TR/css3-color/#hsl-color
3196
+ """
3197
+
3198
+ h = h % 1.0
3199
+ s = min(max(s, 0.0), 1.0)
3200
+ l = min(max(l, 0.0), 1.0)
3201
+
3202
+ if l <= 0.5:
3203
+ m2 = l * (s + 1.0)
3204
+ else:
3205
+ m2 = l + s - l * s
3206
+ m1 = l * 2.0 - m2
3207
+ r = self._hue_to_rgb(m1, m2, h + 1.0 / 3.0)
3208
+ g = self._hue_to_rgb(m1, m2, h)
3209
+ b = self._hue_to_rgb(m1, m2, h - 1.0 / 3.0)
3210
+
3211
+ # Apply gamma correction
3212
+ r **= self.gamma
3213
+ g **= self.gamma
3214
+ b **= self.gamma
3215
+
3216
+ return (r, g, b)
3217
+
3218
+ def _hue_to_rgb(self, m1, m2, h):
3219
+ if h < 0.0:
3220
+ h += 1.0
3221
+ elif h > 1.0:
3222
+ h -= 1.0
3223
+ if h * 6 < 1.0:
3224
+ return m1 + (m2 - m1) * h * 6.0
3225
+ elif h * 2 < 1.0:
3226
+ return m2
3227
+ elif h * 3 < 2.0:
3228
+ return m1 + (m2 - m1) * (2.0 / 3.0 - h) * 6.0
3229
+ else:
3230
+ return m1
3231
+
3232
+
3233
+ TEMPERATURE_COLORMAP = Theme(
3234
+ mincolor=(2.0 / 3.0, 0.80, 0.25), # dark blue
3235
+ maxcolor=(0.0, 1.0, 0.5), # satured red
3236
+ gamma=1.0,
3237
+ )
3238
+
3239
+ PINK_COLORMAP = Theme(
3240
+ mincolor=(0.0, 1.0, 0.90), # pink
3241
+ maxcolor=(0.0, 1.0, 0.5), # satured red
3242
+ )
3243
+
3244
+ GRAY_COLORMAP = Theme(
3245
+ mincolor=(0.0, 0.0, 0.85), # light gray
3246
+ maxcolor=(0.0, 0.0, 0.0), # black
3247
+ )
3248
+
3249
+ BW_COLORMAP = Theme(
3250
+ minfontsize=8.0,
3251
+ maxfontsize=24.0,
3252
+ mincolor=(0.0, 0.0, 0.0), # black
3253
+ maxcolor=(0.0, 0.0, 0.0), # black
3254
+ minpenwidth=0.1,
3255
+ maxpenwidth=8.0,
3256
+ )
3257
+
3258
+ PRINT_COLORMAP = Theme(
3259
+ minfontsize=18.0,
3260
+ maxfontsize=30.0,
3261
+ fontcolor="black",
3262
+ nodestyle="solid",
3263
+ mincolor=(0.0, 0.0, 0.0), # black
3264
+ maxcolor=(0.0, 0.0, 0.0), # black
3265
+ minpenwidth=0.1,
3266
+ maxpenwidth=8.0,
3267
+ )
3268
+
3269
+
3270
+ themes = {
3271
+ "color": TEMPERATURE_COLORMAP,
3272
+ "pink": PINK_COLORMAP,
3273
+ "gray": GRAY_COLORMAP,
3274
+ "bw": BW_COLORMAP,
3275
+ "print": PRINT_COLORMAP,
3276
+ }
3277
+
3278
+
3279
+ def sorted_iteritems(d):
3280
+ # Used mostly for result reproducibility (while testing.)
3281
+ keys = compat_keys(d)
3282
+ keys.sort()
3283
+ for key in keys:
3284
+ value = d[key]
3285
+ yield key, value
3286
+
3287
+
3288
+ class DotWriter:
3289
+ """Writer for the DOT language.
3290
+
3291
+ See also:
3292
+ - "The DOT Language" specification
3293
+ http://www.graphviz.org/doc/info/lang.html
3294
+ """
3295
+
3296
+ strip = False
3297
+ wrap = False
3298
+
3299
+ def __init__(self, fp):
3300
+ self.fp = fp
3301
+
3302
+ def wrap_function_name(self, name):
3303
+ """Split the function name on multiple lines."""
3304
+
3305
+ if len(name) > 32:
3306
+ ratio = 2.0 / 3.0
3307
+ height = max(int(len(name) / (1.0 - ratio) + 0.5), 1)
3308
+ width = max(len(name) / height, 32)
3309
+ # TODO: break lines in symbols
3310
+ name = textwrap.fill(name, width, break_long_words=False)
3311
+
3312
+ # Take away spaces
3313
+ name = name.replace(", ", ",")
3314
+ name = name.replace("> >", ">>")
3315
+ name = name.replace("> >", ">>") # catch consecutive
3316
+
3317
+ return name
3318
+
3319
+ show_function_events = [TOTAL_TIME_RATIO, TIME_RATIO]
3320
+ show_edge_events = [TOTAL_TIME_RATIO, CALLS]
3321
+
3322
+ def graph(self, profile, theme):
3323
+ self.begin_graph()
3324
+
3325
+ fontname = theme.graph_fontname()
3326
+ fontcolor = theme.graph_fontcolor()
3327
+ nodestyle = theme.node_style()
3328
+
3329
+ self.attr("graph", fontname=fontname, ranksep=0.25, nodesep=0.125)
3330
+ self.attr(
3331
+ "node",
3332
+ fontname=fontname,
3333
+ shape="box",
3334
+ style=nodestyle,
3335
+ fontcolor=fontcolor,
3336
+ width=0,
3337
+ height=0,
3338
+ )
3339
+ self.attr("edge", fontname=fontname)
3340
+
3341
+ for _, function in sorted_iteritems(profile.functions):
3342
+ labels = []
3343
+ if function.process is not None:
3344
+ labels.append(function.process)
3345
+ if function.module is not None:
3346
+ labels.append(function.module)
3347
+
3348
+ if self.strip:
3349
+ function_name = function.stripped_name()
3350
+ else:
3351
+ function_name = function.name
3352
+
3353
+ # dot can't parse quoted strings longer than YY_BUF_SIZE, which
3354
+ # defaults to 16K. But some annotated C++ functions (e.g., boost,
3355
+ # https://github.com/jrfonseca/gprof2dot/issues/30) can exceed that
3356
+ MAX_FUNCTION_NAME = 4096
3357
+ if len(function_name) >= MAX_FUNCTION_NAME:
3358
+ sys.stderr.write(
3359
+ "warning: truncating function name with %u chars (%s)\n"
3360
+ % (len(function_name), function_name[:32] + "...")
3361
+ )
3362
+ function_name = function_name[: MAX_FUNCTION_NAME - 1] + unichr(0x2026)
3363
+
3364
+ if self.wrap:
3365
+ function_name = self.wrap_function_name(function_name)
3366
+ labels.append(function_name)
3367
+
3368
+ for event in self.show_function_events:
3369
+ if event in function.events:
3370
+ label = event.format(function[event])
3371
+ labels.append(label)
3372
+ if function.called is not None:
3373
+ labels.append("%u%s" % (function.called, MULTIPLICATION_SIGN))
3374
+
3375
+ if function.weight is not None:
3376
+ weight = function.weight
3377
+ else:
3378
+ weight = 0.0
3379
+
3380
+ label = "\n".join(labels)
3381
+ self.node(
3382
+ function.id,
3383
+ label=label,
3384
+ color=self.color(theme.node_bgcolor(weight)),
3385
+ fontcolor=self.color(theme.node_fgcolor(weight)),
3386
+ fontsize="%.2f" % theme.node_fontsize(weight),
3387
+ tooltip=function.filename,
3388
+ )
3389
+
3390
+ for _, call in sorted_iteritems(function.calls):
3391
+ callee = profile.functions[call.callee_id]
3392
+
3393
+ labels = []
3394
+ for event in self.show_edge_events:
3395
+ if event in call.events:
3396
+ label = event.format(call[event])
3397
+ labels.append(label)
3398
+
3399
+ if call.weight is not None:
3400
+ weight = call.weight
3401
+ elif callee.weight is not None:
3402
+ weight = callee.weight
3403
+ else:
3404
+ weight = 0.0
3405
+
3406
+ label = "\n".join(labels)
3407
+
3408
+ self.edge(
3409
+ function.id,
3410
+ call.callee_id,
3411
+ label=label,
3412
+ color=self.color(theme.edge_color(weight)),
3413
+ fontcolor=self.color(theme.edge_color(weight)),
3414
+ fontsize="%.2f" % theme.edge_fontsize(weight),
3415
+ penwidth="%.2f" % theme.edge_penwidth(weight),
3416
+ labeldistance="%.2f" % theme.edge_penwidth(weight),
3417
+ arrowsize="%.2f" % theme.edge_arrowsize(weight),
3418
+ )
3419
+
3420
+ self.end_graph()
3421
+
3422
+ def begin_graph(self):
3423
+ self.write("digraph {\n")
3424
+
3425
+ def end_graph(self):
3426
+ self.write("}\n")
3427
+
3428
+ def attr(self, what, **attrs):
3429
+ self.write("\t")
3430
+ self.write(what)
3431
+ self.attr_list(attrs)
3432
+ self.write(";\n")
3433
+
3434
+ def node(self, node, **attrs):
3435
+ self.write("\t")
3436
+ self.id(node)
3437
+ self.attr_list(attrs)
3438
+ self.write(";\n")
3439
+
3440
+ def edge(self, src, dst, **attrs):
3441
+ self.write("\t")
3442
+ self.id(src)
3443
+ self.write(" -> ")
3444
+ self.id(dst)
3445
+ self.attr_list(attrs)
3446
+ self.write(";\n")
3447
+
3448
+ def attr_list(self, attrs):
3449
+ if not attrs:
3450
+ return
3451
+ self.write(" [")
3452
+ first = True
3453
+ for name, value in sorted_iteritems(attrs):
3454
+ if value is None:
3455
+ continue
3456
+ if first:
3457
+ first = False
3458
+ else:
3459
+ self.write(", ")
3460
+ self.id(name)
3461
+ self.write("=")
3462
+ self.id(value)
3463
+ self.write("]")
3464
+
3465
+ def id(self, id):
3466
+ if isinstance(id, (int, float)):
3467
+ s = str(id)
3468
+ elif isinstance(id, basestring):
3469
+ if id.isalnum() and not id.startswith("0x"):
3470
+ s = id
3471
+ else:
3472
+ s = self.escape(id)
3473
+ else:
3474
+ raise TypeError
3475
+ self.write(s)
3476
+
3477
+ def color(self, rgb):
3478
+ r, g, b = rgb
3479
+
3480
+ def float2int(f):
3481
+ if f <= 0.0:
3482
+ return 0
3483
+ if f >= 1.0:
3484
+ return 255
3485
+ return int(255.0 * f + 0.5)
3486
+
3487
+ return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)])
3488
+
3489
+ def escape(self, s):
3490
+ if not PYTHON_3:
3491
+ s = s.encode("utf-8")
3492
+ s = s.replace("\\", r"\\")
3493
+ s = s.replace("\n", r"\n")
3494
+ s = s.replace("\t", r"\t")
3495
+ s = s.replace('"', r"\"")
3496
+ return '"' + s + '"'
3497
+
3498
+ def write(self, s):
3499
+ self.fp.write(s)
3500
+
3501
+
3502
+ ########################################################################
3503
+ # Main program
3504
+
3505
+
3506
+ def naturalJoin(values):
3507
+ if len(values) >= 2:
3508
+ return ", ".join(values[:-1]) + " or " + values[-1]
3509
+
3510
+ else:
3511
+ return "".join(values)
3512
+
3513
+
3514
+ def main(argv=sys.argv[1:]):
3515
+ """Main program."""
3516
+
3517
+ global totalMethod
3518
+
3519
+ formatNames = list(formats.keys())
3520
+ formatNames.sort()
3521
+
3522
+ themeNames = list(themes.keys())
3523
+ themeNames.sort()
3524
+
3525
+ labelNames = list(labels.keys())
3526
+ labelNames.sort()
3527
+
3528
+ optparser = optparse.OptionParser(usage="\n\t%prog [options] [file] ...")
3529
+ optparser.add_option(
3530
+ "-o",
3531
+ "--output",
3532
+ metavar="FILE",
3533
+ type="string",
3534
+ dest="output",
3535
+ help="output filename [stdout]",
3536
+ )
3537
+ optparser.add_option(
3538
+ "-n",
3539
+ "--node-thres",
3540
+ metavar="PERCENTAGE",
3541
+ type="float",
3542
+ dest="node_thres",
3543
+ default=0.5,
3544
+ help="eliminate nodes below this threshold [default: %default]",
3545
+ )
3546
+ optparser.add_option(
3547
+ "-e",
3548
+ "--edge-thres",
3549
+ metavar="PERCENTAGE",
3550
+ type="float",
3551
+ dest="edge_thres",
3552
+ default=0.1,
3553
+ help="eliminate edges below this threshold [default: %default]",
3554
+ )
3555
+ optparser.add_option(
3556
+ "-f",
3557
+ "--format",
3558
+ type="choice",
3559
+ choices=formatNames,
3560
+ dest="format",
3561
+ default="prof",
3562
+ help="profile format: %s [default: %%default]" % naturalJoin(formatNames),
3563
+ )
3564
+ optparser.add_option(
3565
+ "--total",
3566
+ type="choice",
3567
+ choices=("callratios", "callstacks"),
3568
+ dest="totalMethod",
3569
+ default=totalMethod,
3570
+ help="preferred method of calculating total time: callratios or callstacks (currently affects only perf format) [default: %default]",
3571
+ )
3572
+ optparser.add_option(
3573
+ "-c",
3574
+ "--colormap",
3575
+ type="choice",
3576
+ choices=themeNames,
3577
+ dest="theme",
3578
+ default="color",
3579
+ help="color map: %s [default: %%default]" % naturalJoin(themeNames),
3580
+ )
3581
+ optparser.add_option(
3582
+ "-s",
3583
+ "--strip",
3584
+ action="store_true",
3585
+ dest="strip",
3586
+ default=False,
3587
+ help="strip function parameters, template parameters, and const modifiers from demangled C++ function names",
3588
+ )
3589
+ optparser.add_option(
3590
+ "--color-nodes-by-selftime",
3591
+ action="store_true",
3592
+ dest="color_nodes_by_selftime",
3593
+ default=False,
3594
+ help="color nodes by self time, rather than by total time (sum of self and descendants)",
3595
+ )
3596
+ optparser.add_option(
3597
+ "--colour-nodes-by-selftime",
3598
+ action="store_true",
3599
+ dest="color_nodes_by_selftime",
3600
+ help=optparse.SUPPRESS_HELP,
3601
+ )
3602
+ optparser.add_option(
3603
+ "-w",
3604
+ "--wrap",
3605
+ action="store_true",
3606
+ dest="wrap",
3607
+ default=False,
3608
+ help="wrap function names",
3609
+ )
3610
+ optparser.add_option(
3611
+ "--show-samples",
3612
+ action="store_true",
3613
+ dest="show_samples",
3614
+ default=False,
3615
+ help="show function samples",
3616
+ )
3617
+ optparser.add_option(
3618
+ "--node-label",
3619
+ metavar="MEASURE",
3620
+ type="choice",
3621
+ choices=labelNames,
3622
+ action="append",
3623
+ dest="node_labels",
3624
+ help="measurements to on show the node (can be specified multiple times): %s [default: %s]"
3625
+ % (naturalJoin(labelNames), ", ".join(defaultLabelNames)),
3626
+ )
3627
+ # add option to show information on available entries ()
3628
+ optparser.add_option(
3629
+ "--list-functions",
3630
+ type="string",
3631
+ dest="list_functions",
3632
+ default=None,
3633
+ help="""\
3634
+ list functions available for selection in -z or -l, requires selector argument
3635
+ ( use '+' to select all).
3636
+ Recall that the selector argument is used with Unix/Bash globbing/pattern matching,
3637
+ and that entries are formatted '<pkg>:<linenum>:<function>'. When argument starts
3638
+ with '%', a dump of all available information is performed for selected entries,
3639
+ after removal of leading '%'.
3640
+ """,
3641
+ )
3642
+ # add option to create subtree or show paths
3643
+ optparser.add_option(
3644
+ "-z",
3645
+ "--root",
3646
+ type="string",
3647
+ dest="root",
3648
+ default="",
3649
+ help="prune call graph to show only descendants of specified root function",
3650
+ )
3651
+ optparser.add_option(
3652
+ "-l",
3653
+ "--leaf",
3654
+ type="string",
3655
+ dest="leaf",
3656
+ default="",
3657
+ help="prune call graph to show only ancestors of specified leaf function",
3658
+ )
3659
+ optparser.add_option(
3660
+ "--depth",
3661
+ type="int",
3662
+ dest="depth",
3663
+ default=-1,
3664
+ help="prune call graph to show only descendants or ancestors until specified depth",
3665
+ )
3666
+ # add a new option to control skew of the colorization curve
3667
+ optparser.add_option(
3668
+ "--skew",
3669
+ type="float",
3670
+ dest="theme_skew",
3671
+ default=1.0,
3672
+ help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Values > 1.0 give less variety to lower percentages",
3673
+ )
3674
+ # add option for filtering by file path
3675
+ optparser.add_option(
3676
+ "-p",
3677
+ "--path",
3678
+ action="append",
3679
+ type="string",
3680
+ dest="filter_paths",
3681
+ help="Filter all modules not in a specified path",
3682
+ )
3683
+ (options, args) = optparser.parse_args(argv)
3684
+
3685
+ if len(args) > 1 and options.format != "pstats":
3686
+ optparser.error("incorrect number of arguments")
3687
+
3688
+ try:
3689
+ theme = themes[options.theme]
3690
+ except KeyError:
3691
+ optparser.error("invalid colormap '%s'" % options.theme)
3692
+
3693
+ # set skew on the theme now that it has been picked.
3694
+ if options.theme_skew:
3695
+ theme.skew = options.theme_skew
3696
+
3697
+ totalMethod = options.totalMethod
3698
+
3699
+ try:
3700
+ Format = formats[options.format]
3701
+ except KeyError:
3702
+ optparser.error("invalid format '%s'" % options.format)
3703
+
3704
+ if Format.stdinInput:
3705
+ if not args:
3706
+ fp = sys.stdin
3707
+ elif PYTHON_3:
3708
+ fp = open(args[0], "rt", encoding="UTF-8")
3709
+ else:
3710
+ fp = open(args[0], "rt")
3711
+ parser = Format(fp)
3712
+ elif Format.multipleInput:
3713
+ if not args:
3714
+ optparser.error(
3715
+ "at least a file must be specified for %s input" % options.format
3716
+ )
3717
+ parser = Format(*args)
3718
+ else:
3719
+ if len(args) != 1:
3720
+ optparser.error(
3721
+ "exactly one file must be specified for %s input" % options.format
3722
+ )
3723
+ parser = Format(args[0])
3724
+
3725
+ profile = parser.parse()
3726
+
3727
+ if options.output is None:
3728
+ if PYTHON_3:
3729
+ output = open(
3730
+ sys.stdout.fileno(), mode="wt", encoding="UTF-8", closefd=False
3731
+ )
3732
+ else:
3733
+ output = sys.stdout
3734
+ else:
3735
+ if PYTHON_3:
3736
+ output = open(options.output, "wt", encoding="UTF-8")
3737
+ else:
3738
+ output = open(options.output, "wt")
3739
+
3740
+ dot = DotWriter(output)
3741
+ dot.strip = options.strip
3742
+ dot.wrap = options.wrap
3743
+
3744
+ labelNames = options.node_labels or defaultLabelNames
3745
+ dot.show_function_events = [labels[l] for l in labelNames]
3746
+ if options.show_samples:
3747
+ dot.show_function_events.append(SAMPLES)
3748
+
3749
+ profile = profile
3750
+ profile.prune(
3751
+ options.node_thres / 100.0,
3752
+ options.edge_thres / 100.0,
3753
+ options.filter_paths,
3754
+ options.color_nodes_by_selftime,
3755
+ )
3756
+
3757
+ if options.list_functions:
3758
+ profile.printFunctionIds(selector=options.list_functions)
3759
+ sys.exit(0)
3760
+
3761
+ if options.root:
3762
+ rootIds = profile.getFunctionIds(options.root)
3763
+ if not rootIds:
3764
+ sys.stderr.write(
3765
+ "root node "
3766
+ + options.root
3767
+ + " not found (might already be pruned : try -e0 -n0 flags)\n"
3768
+ )
3769
+ sys.exit(1)
3770
+ profile.prune_root(rootIds, options.depth)
3771
+ if options.leaf:
3772
+ leafIds = profile.getFunctionIds(options.leaf)
3773
+ if not leafIds:
3774
+ sys.stderr.write(
3775
+ "leaf node "
3776
+ + options.leaf
3777
+ + " not found (maybe already pruned : try -e0 -n0 flags)\n"
3778
+ )
3779
+ sys.exit(1)
3780
+ profile.prune_leaf(leafIds, options.depth)
3781
+
3782
+ dot.graph(profile, theme)
3783
+
3784
+
3785
+ if __name__ == "__main__":
3786
+ main()