shaka-player 4.0.2 → 4.0.5

package/build/stats.py

@@ -1,816 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2016 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""A program for analyzing the Shaka compiled sources.
-
-This can be used to find areas that can be removed if not needed.  This uses
-the source map (i.e. shaka-player.compiled.debug.map) to find the compiled code
-size, see:
-    https://github.com/mattrobenolt/python-sourcemap
-    http://www.html5rocks.com/en/tutorials/developertools/sourcemaps/
-
-This script can output four different stats in two different formats:
-- The size of functions and namespaces.
-- The dependencies between types (in plain or DOT format).
-- The dependencies between functions (in plain or DOT format).
-- All tokens in the source map.
-
-The dependencies can be outputted in DOT format which can be used with graph
-programs to display a visual layout of the dependencies.  For example, using
-graphviz:
-
-  ./stats.py -c -d | fdb -Goverlap=prism | neato -n2 -Tsvg > out.svg
-"""
-
-from __future__ import print_function
-
-import argparse
-import json
-import logging
-import math
-import os
-import string
-
-import shakaBuildHelpers
-
-
-# A Base64 VLQ digit can represent 5 bits, so it is Base32.
-VLQ_BASE_SHIFT = 5
-VLQ_BASE = 1 << VLQ_BASE_SHIFT
-
-# A mask of bits for a VLQ digit (11111), 31 decimal
-VLQ_BASE_MASK = VLQ_BASE - 1
-
-# The continuation bit is the 6th bit
-VLQ_CONTINUATION_BIT = VLQ_BASE
-
-# Don't use Base64 lib since it is not a real Base64 string; it simply
-# decodes each character to a single Base64 number.
-B64 = dict((c, i) for i, c in
-           enumerate('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
-                     '0123456789+/'))
-
-
-def from_vlq_signed(value):
-  """Converts a VLQ number to a normal signed number.
-
-  Args:
-    value: A number decoded from a VLQ string.
-
-  Returns:
-    an integer.
-  """
-  negative = (value & 1) == 1
-  value >>= 1
-  return -value if negative else value
-
-
-class Segment(object):
-  """Defines an entry in the source map.
-
-  Members:
-    dst_col_offset - The offset of the destination column from the previous
-      segment.
-    name_offset - If not None, the offset of the name index from the previous
-      segment.
-  """
-
-  def __init__(self, data):
-    self.dst_col_offset = data[0]
-    self.name_offset = data[4] if len(data) > 4 else None
-
-
-def decode_segment(segment):
-  """Decodes VLQ values from the given segment.
-
-  Args:
-    segment: A string containing the encoded segment text.
-
-  Returns:
-    the parsed Segment.
-  """
-
-  values = []
-  cur, shift = 0, 0
-
-  for c in segment:
-    digit = B64[c]
-    cont = (digit & VLQ_CONTINUATION_BIT) != 0
-    digit &= VLQ_BASE_MASK
-    cur += digit << shift
-    shift += VLQ_BASE_SHIFT
-
-    if not cont:
-      values.append(from_vlq_signed(cur))
-      cur, shift = 0, 0
-
-  # A valid VLQ string should not have dangling bits.
-  assert cur == 0
-  assert shift == 0
-  return Segment(values)
-
-
-class Token(object):
-  """A Token represents one JavaScript symbol.
-
-  For example, this can be a variable or an equals sign.  If this is a variable
-  or the keyword 'function' it will usually have a name which indicates what it
-  originally was defined. But there are also tokens such as ; and ( which appear
-  as tokens in the map but to not have explicit name (see isFunction).
-
-  Members:
-      dst_line - Line index in compiled code
-      dst_col - Column index in compiled code
-      name - Name of the token; or None
-  """
-
-  def __init__(self, dst_line, dst_col, name=None):
-    self.dst_line = dst_line
-    self.dst_col = dst_col
-    self.name = name
-
-  def __str__(self):
-    return str(self.name)
-
-
-def decode_mappings(line_data, names):
-  """Decodes a mappings line of text.
-
-  Args:
-    line_data: A string containing the mapping line.
-    names: An array of strings containing the names of the objects.
-
-  Returns:
-    a list of Tokens
-  """
-
-  tokens = []
-
-  lines = line_data.split(';')
-  name_id = 0
-  for dst_line, line in enumerate(lines):
-    dst_col = 0
-    segments = line.split(',')
-    for segment in segments:
-      if not segment:
-        continue
-
-      segment = decode_segment(segment)
-      dst_col += segment.dst_col_offset
-
-      # segment.dst_col can be negative (more useful in names below); however
-      # after applying a negative offset, the result must still be positive.
-      assert dst_col >= 0
-
-      name = None
-      if segment.name_offset is not None:
-        name_id += segment.name_offset
-        assert name_id >= 0
-        name = names[name_id]
-
-      tokens.append(Token(dst_line, dst_col, name))
-
-  return tokens
-
-
-def is_function(token, lines):
-  """Determines if the given token is the start of a function.
-
-  All function definitions are assumed to have a name field and the token in
-  the compiled source is the keyword 'function'. Sometimes the function is
-  defined on the previous semicolon and sometimes that semicolon appears on
-  the previous line.
-
-  Args:
-    token: The Token to check.
-    lines: An array of compiled code lines.
-
-  Returns:
-    whether the token is a function.
-  """
-
-  # All functions have a name.
-  if not token.name:
-    return False
-
-  # Sometimes a function token starts with the previous ;
-  # Also sometimes the token starts on the ; that is on the previous
-  # line.
-  partial_line = lines[token.dst_line][token.dst_col:]
-  if partial_line == ';\n':
-    if len(lines) == token.dst_line + 1:
-      return False
-    else:
-      return lines[token.dst_line + 1].startswith('function')
-  else:
-    return (partial_line.startswith('function') or
-            partial_line.startswith(';function'))
-
-
-def read_function(token_iter, prev, prev_index, lines, callback):
-  """Reads a function from the token stream.
-
-  The function token should already be consumed.
-
-  Args:
-    token_iter: An iterator of the tokens.
-    prev: The token containing the function definition.
-    prev_index: The index of the previous token.
-    lines: An array of compiled code lines.
-    callback: A callback type used to create the data.  See traverse_tokens.
-
-  Returns:
-    an array of State objects in a format controlled by the callback (see
-    traverse_tokens).
-  """
-
-  brackets = 0
-  read = False
-  ret = []
-
-  partial_line = lines[prev.dst_line][prev.dst_col:]
-  state = callback(prev, prev_index)
-
-  try:
-    while not read or brackets > 0:
-      index, token = next(token_iter)
-      partial_line = lines[token.dst_line][token.dst_col:]
-
-      # Recursively read functions.  Sometimes functions are defined nested.
-      # This doesn't happen that often, and never for Shaka methods, so it does
-      # not count it twice since the size of this method includes the nested
-      # function.
-      if is_function(token, lines):
-        ret += read_function(token_iter, token, index, lines, callback)
-      else:
-        state.add(token, index)
-
-      if partial_line.startswith('{}'):
-        read = True
-      elif partial_line[0] == '{':
-        brackets += 1
-        read = True
-      elif partial_line[0] == '}':
-        brackets -= 1
-  # When we run out of tokens, simply ignore it.  A parent call will not see
-  # this error; but it will continue and the next call to 'next' will fail
-  # with another StopIteration.  This ensures that the last State object
-  # is included for invalid content.
-  except StopIteration:
-    pass
-
-  temp = state.build()
-  if temp:
-    ret.append(temp)
-
-  return ret
-
-
-def traverse_tokens(tokens, lines, callback):
-  """Traverses a list of tokens to identify functions.
-
-  Then uses a callback to perform some work on the functions.  Each function
-  seen gets a new State object created from the given callback method; there is
-  a single State for global code which is given None in the constructor.  Then,
-  each token seen is passed to the 'add' method of the State.  This is used by
-  the State to either calculate sizes, print tokens, or detect dependencies.
-  The 'build' method is called at the end of the function to create a result
-  object that is returned as an array at the end.
-
-  Args:
-    tokens: An array of Tokens.
-    lines: An array of compiled code lines.
-    callback: A constructor that returns a state object.  It takes a start
-        token or None if outside a function.  It has two member functions
-          add - accepts the current token and the token's index.
-          build - returns an object to be added to the results.
-
-  Returns:
-    an array of State objects in a format controlled by the callback.
-  """
-
-  ret = []
-  state = callback(None, None)
-  # Create a token iterator.  This is used to read tokens from the array.  We
-  # cannot use a for loop because the iterator is passed to readFunction.
-  token_iter = enumerate(tokens)
-  try:
-    while True:
-      index, token = next(token_iter)
-
-      if is_function(token, lines):
-        ret += read_function(token_iter, token, index, lines, callback)
-      else:
-        state.add(token, index)
-  except StopIteration:
-    pass
-
-  temp = state.build()
-  if temp:
-    ret.append(temp)
-  return ret
-
-
-class FunctionSize(object):
-  """Contains information about a function's size."""
-
-  def __init__(self, name, size):
-    self.name = name
-    self.size = size
-
-
-def print_tokens(tokens, lines, funcs):
-  """Prints the given tokens.
-
-  Args:
-    tokens: An array of Tokens.
-    lines: An array of compiled code lines.
-    funcs: An array of FunctionSize.
-  """
-
-  class State(object):
-    """Defines the current parser state."""
-
-    def __init__(self, token, index):
-      # The start of a function, or the global start.
-      self.name = token.name if token else None
-      if token:
-        self._print_token('>', token, index)
-
-    def _print_token(self, prefix, token, index):
-      partial_line = lines[token.dst_line][token.dst_col:]
-      if len(tokens) > index + 1:
-        next_ = tokens[index + 1]
-        if next_.dst_line == token.dst_line:
-          partial_line = lines[token.dst_line][token.dst_col:next_.dst_col]
-      token_text = partial_line[:10].replace('\n', '').rjust(12)
-      print('%s %4d %4d %12s %s' % (prefix, token.dst_line, token.dst_col,
-                                    token_text, token.name))
-
-    def add(self, token, index):
-      """Parses the given token.
-
-      Args:
-        token: The token to add.
-        index: The index of the token in the original array.
-      """
-      prefix = None
-      if not self.name:
-        prefix = '!'
-      elif lines[token.dst_line][token.dst_col:token.dst_col+2] == '{}':
-        prefix = ' '
-      elif lines[token.dst_line][token.dst_col] == '{':
-        prefix = '+'
-      elif lines[token.dst_line][token.dst_col] == '}':
-        prefix = '-'
-      else:
-        prefix = ' '
-
-      self._print_token(prefix, token, index)
-
-    def build(self):
-      if not self.name:
-        return
-
-      # The end of a function.  Print the size of this function.
-      size = 0
-      this_func = [t for t in funcs if t.name == self.name]
-      if this_func:
-        size = this_func[0].size
-      print('X %s %d' % (self.name, size))
-
-  traverse_tokens(tokens, lines, State)
-
-
-class FunctionDependencies(object):
-  """Contains information about a function's dependencies."""
-
-  def __init__(self, name, deps):
-    self.name = name
-    self.deps = deps
-
-
-def process_deps(tokens, lines, is_class):
-  """Processes the tokens into function or class dependencies.
-
-  Args:
-    tokens: An array of Tokens.
-    lines: An array of compiled code lines.
-    is_class: Whether to create a class graph instead of a function graph.
-
-  Returns:
-    an array of FunctionDependencies.
-  """
-
-  class State(object):
-    """Defines the current parser state."""
-
-    def __init__(self, token, _):
-      self.deps = []
-      self.name, self.parts = self._create_parts(token)
-
-    def _create_parts(self, token):
-      """Creates an array of name parts.
-
-      Args:
-        token: The token to create the name from.
-
-      Returns:
-        A tuple of the name and the array of name parts.
-      """
-      if not token or not token.name:
-        return (None, None)
-      parts = token.name.split('.')
-      name = token.name
-
-      # Instance methods are the same as static methods.
-      if len(parts) > 2 and parts[-2] == 'prototype':
-        del parts[-2]
-
-      # Strip function names if class graph; also remove it from the name.
-      if is_class:
-        if parts[-1][0] in string.ascii_lowercase:
-          del parts[-1]
-          name = '.'.join(parts)
-
-      return (name, parts)
-
-    def add(self, token, _):
-      """Parses the given token.
-
-      Args:
-        token: The token to parse.
-      """
-      # Ignore symbols outside a function.  Only care about function
-      # references and only those that reference our code.
-      if not self.name or not token.name or not token.name.startswith('shaka.'):
-        return
-
-      name, other_parts = self._create_parts(token)
-
-      # Get the index of the first different namespace.
-      count = min(len(self.parts), len(other_parts))
-      i = 0
-      while i < count and self.parts[i] == other_parts[i]:
-        i += 1
-
-      # Ignore use of members of the same object:
-      # OfflineVideoSource.configure and OfflineVideoSource.store
-      if (i == count - 1 or i == count) and len(self.parts) == len(other_parts):
-        return
-
-      # Ignore use of the constructor of the same type:
-      # OfflineVideoSource and OfflineVideoSource.store
-      if i == count and abs(len(self.parts) - len(other_parts)) == 1:
-        return
-
-      # Add the dependency.
-      if name not in self.deps:
-        self.deps.append(name)
-
-    def build(self):
-      return FunctionDependencies(self.name, self.deps) if self.name else None
-
-  ret = traverse_tokens(tokens, lines, State)
-  assert ret
-  ret = sorted(ret, key=lambda key: key.name)
-
-  # We need to collapse duplicates.
-  i = 0
-  while i + 1 < len(ret):
-    if ret[i].name == ret[i + 1].name:
-      for dep in ret[i + 1].deps:
-        if dep not in ret[i].deps:
-          ret[i].deps.append(dep)
-      del ret[i + 1]
-    else:
-      i += 1
-
-  return ret
-
-
-def process_sizes(tokens, lines):
-  """Processes an array of tokens into function lengths.
-
-  Args:
-    tokens: An array of Tokens.
-    lines: An array of compiled code lines.
-
-  Returns:
-    an array of FunctionSizes sorted on name.
-  """
-
-  class State(object):
-    """Defines the current parser state."""
-
-    def __init__(self, token, _):
-      self.name = token.name if token else None
-      self.size = 0
-      self.start = token.dst_col if token else None
-      self.line = token.dst_line if token else None
-
-    def add(self, token, _):
-      """Parses the given token.
-
-      Args:
-        token: The token to parse.
-      """
-      # Ignore outside a function
-      if not self.name:
-        return
-      # If we skipped to the next line, include the code to the end of the line.
-      # If we skipped multiple lines, include the whole line.  This will most
-      # likely never happen since the compiled code usually has new lines on
-      # function boundaries.
-      assert token.dst_line >= self.line
-      while token.dst_line != self.line:
-        self.size += len(lines[self.line]) - self.start
-        self.line += 1
-        self.start = 0
-
-      # Keep increasing the size.  We can't simply keep the start and measure
-      # at the end since we are not given the end token in build().
-      self.size += token.dst_col - self.start
-      self.start = token.dst_col
-
-    def build(self):
-      return FunctionSize(self.name, self.size) if self.name else None
-
-  ret = traverse_tokens(tokens, lines, State)
-  assert ret
-
-  ret = [k for k in ret if k.name and
-         (k.name.startswith('shaka.') or k.name.startswith('goog.'))]
-  ret = sorted(ret, key=lambda key: key.name)
-
-  # We need to collapse duplicates.
-  i = 0
-  while i + 1 < len(ret):
-    if ret[i].name == ret[i + 1].name:
-      ret[i].size += ret[i + 1].size
-      del ret[i + 1]
-    else:
-      i += 1
-
-  return ret
-
-
-def print_tree(results, indent, callback, end_callback):
-  """Prints the results in an indented format.
-
-  Args:
-    results: An array of FunctionSizes sorted on name.
-    indent: A number to indent.
-    callback: A callback function to print the data.  Accepts a title, an
-        indentation, and a sublist of the items in that group.
-    end_callback: A callback function called after a group; can be None.
-  """
-
-  # This is used both when printing sizes and when printing dependencies in
-  # DOT format.  This recursively creates groups of items with the same prefix.
-  # e.g.
-  # shaka
-  #  shaka.util
-  #   shaka.util.FailoverUri
-  #   shaka.util.TypedBind
-  #  shaka.player
-  # ...
-
-  if len(results) <= 1:
-    callback(None, indent, results)
-    return
-
-  # We want to group-by prefixes for the elements.  Since it is sorted, we
-  # can find the overall prefix length.
-  first = results[0].name.split('.')
-  last = results[-1].name.split('.')
-  prefix = 0
-  while (prefix < len(first) and prefix < len(last)
-         and first[prefix] == last[prefix]):
-    prefix += 1
-
-  group = 0
-  group_items = first
-  if prefix == len(first):
-    # This happens when the group has a first element of a class name and the
-    # remaining are member functions.  Remove the first element from this
-    # group.
-    group_items = results[1].name.split('.')
-    group = 1
-
-  # Start with second element, and go one more so we make sure to process the
-  # last group.
-  for i in range(1, len(results) + 1):
-    if i == len(results):
-      items = [''] * (prefix + 1)
-    else:
-      items = results[i].name.split('.')
-    if items[prefix] != group_items[prefix]:
-      title = '.'.join(group_items[:(prefix + 1)])
-      callback(title, indent, results[group:i])
-
-      print_tree(results[group:i], indent + 1, callback, end_callback)
-
-      # Set the start of the next group to the current element.
-      group = i
-      group_items = items
-
-  if end_callback:
-    end_callback(indent)
-
-
-def print_sizes(sizes):
-  """Prints the sizes in an indented format.
-
-  Args:
-    sizes: An array of FunctionSizes sorted on name.
-  """
-  # This callback is used to print the total sizes of each of the sub-groups.
-  # Using the indent as padding allows to print a tree-like structure to
-  # show how big each section is.
-  def callback_factory(padding):
-    # Use a factory so we capture the padding.
-    def callback(title, indent, results):
-      if title:
-        size = sum([k.size for k in results])
-        print('%s %*d %s' % (indent * '  ', padding, size, title))
-    return callback
-
-  total = sum([k.size for k in sizes])
-  padding = int(math.ceil(math.log10(total)))
-
-  print('%*d %s' % (padding, total, 'TOTAL'))
-  print_tree(sizes, 0, callback_factory(padding), None)
-
-
-def print_deps(results, in_dot):
-  """Prints the dependencies.
-
-  Arguments:
-    results: A sorted array of FunctionDependencies.
-    in_dot: Whether to print in DOT format.
-  """
-  if not in_dot:
-    for func in results:
-      name, deps = func.name, func.deps
-
-      # Ignore items with no dependencies.
-      if deps:
-        print(name)
-
-      for dep in deps:
-        print('   ' + dep)
-
-    return
-
-  dep_map = dict()
-
-  # Use the printTree to produce clusters for each namespace and type.  This
-  # will print boxes around each class and show dependencies between types.
-  print('digraph {')
-  def callback_factory(dep_map, temp):
-    """Creates a callback function."""
-    def callback(title, indent, results):
-      if title:
-        if len(results) > 1:
-          print('%s subgraph cluster%d {' % ('\t' * indent, len(temp)))
-          temp.append(1)
-        else:
-          print('%s %d [label="%s"];' % ('\t' * indent, len(dep_map),
-                                         results[0].name))
-          dep_map[results[0].name] = len(dep_map)
-
-    return callback
-
-  def end_callback(indent):
-    if indent > 1:
-      print('\t' * (indent - 1), '}')
-
-  print_tree(results, 1, callback_factory(dep_map, []), end_callback)
-
-  for func in results:
-    name, deps = func.name, func.deps
-
-    # Ignore items with no dependencies.
-    if deps:
-      if name not in dep_map:
-        dep_map[name] = len(dep_map)
-        print('\t%s [label="%s"];' % (dep_map[name], name))
-
-    for dep in deps:
-      if dep not in dep_map:
-        dep_map[dep] = len(dep_map)
-        print('\t%s [label="%s"];' % (dep_map[dep], dep))
-
-      print('\t%s -> %s;' % (dep_map[name], dep_map[dep]))
-
-  print('}')
-
-
-def process(text, options):
-  """Decodes a JSON string containing source map data.
-
-  Args:
-    text: A JSON string containing source map data.
-    options: An object containing the command-line options.
-  """
-
-  # The spec allows a map file to start with )]} to prevent javascript from
-  # including it.
-  if text.startswith(')]}\'\n') or text.startswith(')]}\n'):
-    _, text = text.split('\n', 1)
-
-  # Decode the JSON data and get the parts we need.
-  data = json.loads(text)
-  # Paths are relative to the output directory.
-  base = os.path.join(shakaBuildHelpers.get_source_base(), 'dist')
-  with shakaBuildHelpers.open_file(os.path.join(base, data['file']), 'r') as f:
-    file_lines = f.readlines()
-  names = data['names']
-  mappings = data['mappings']
-  tokens = decode_mappings(mappings, names)
-  sizes = process_sizes(tokens, file_lines)
-
-  # Print out one of the results.
-  if options.all_tokens:
-    print_tokens(tokens, file_lines, sizes)
-  elif options.function_sizes:
-    print_sizes(sizes)
-  elif options.function_deps or options.class_deps:
-    temp = process_deps(tokens, file_lines, options.class_deps)
-    print_deps(temp, options.dot_format)
-
-
-def main(args):
-  parser = argparse.ArgumentParser(
-      description=__doc__,
-      formatter_class=argparse.RawDescriptionHelpFormatter)
-
-  parser.add_argument('-d', '--dot-format', action='store_true',
-                      help='Prints in DOT format.')
-  parser.add_argument('source_map', nargs='?',
-                      default='shaka-player.compiled.map',
-                      help='The source map or the name of the build to use.')
-
-  print_types = parser.add_mutually_exclusive_group(required=True)
-  print_types.add_argument('-c', '--class-deps', action='store_true',
-                           help='Prints the class dependencies.')
-  print_types.add_argument('-f', '--function-deps', action='store_true',
-                           help='Prints the function dependencies.')
-  print_types.add_argument(
-      '-s', '--function-sizes', action='store_true',
-      help='Prints the function sizes (in number of characters).')
-  print_types.add_argument(
-      '-t', '--all-tokens', action='store_true',
-      help='Prints all tokens in the source map.')
-
-  options = parser.parse_args(args)
-
-  # Verify arguments are correct.
-  if (options.dot_format and not options.function_deps
-      and not options.class_deps):
-    parser.error('--dot-format only valid with --function-deps or '
-                 '--class-deps.')
-
-  # Try to find the file
-  name = options.source_map
-  if not os.path.isfile(name):
-    # Get the source code base directory
-    base = shakaBuildHelpers.get_source_base()
-
-    # Supports the following searches:
-    # * File name given, map in dist/
-    # * Type given, map in working directory
-    # * Type given, map in dist/
-    if os.path.isfile(os.path.join(base, 'dist', name)):
-      name = os.path.join(base, 'dist', name)
-    elif os.path.isfile(
-        os.path.join('shaka-player.' + name + '.debug.map')):
-      name = os.path.join('shaka-player.' + name + '.debug.map')
-    elif os.path.isfile(
-        os.path.join(base, 'dist', 'shaka-player.' + name + '.debug.map')):
-      name = os.path.join(base, 'dist', 'shaka-player.' + name + '.debug.map')
-    else:
-      logging.error('"%s" not found; build Shaka first.', name)
-      return 1
-
-  with shakaBuildHelpers.open_file(name, 'r') as f:
-    process(f.read(), options)
-  return 0
-
-
-if __name__ == '__main__':
-  shakaBuildHelpers.run_main(main)