emdump.py

#!/usr/bin/env python3
# Copyright 2018 The Emscripten Authors.  All rights reserved.
# Emscripten is available under two separate licenses, the MIT license and the
# University of Illinois/NCSA Open Source License.  Both these licenses can be
# found in the LICENSE file.

# -*- Mode: python -*-

"""emdump.py prints out statistics about compiled code sizes
"""

import argparse
import os
import re
import subprocess
import sys
from pathlib import Path


# If true, we are printing delta information between two data sets. If false, we are just printing symbol info for a single data set
diffing_two_data_sets = False

# Global command line options
options = None


# Finds the given executable 'program' in PATH. Operates like the Unix tool 'which'.
def which(program):
  def is_exe(fpath):
    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

  fpath, fname = os.path.split(program)
  if fpath:
    if is_exe(program):
      return program
  else:
    for path in os.environ["PATH"].split(os.pathsep):
      path = path.strip('"')
      exe_file = os.path.join(path, program)
      if is_exe(exe_file):
        return exe_file

      if os.name == 'nt' and '.' not in fname:
        if is_exe(exe_file + '.exe'):
          return exe_file + '.exe'
        if is_exe(exe_file + '.cmd'):
          return exe_file + '.cmd'
        if is_exe(exe_file + '.bat'):
          return exe_file + '.bat'

  return None


# Given a string s and an index i, counts how many times character ch is repeated looking backwards at s[i], s[i-1], s[i-2], s[i-3], ...
def rcount(s, ch, i):
  j = i
  while j > 0 and s[j] == ch:
    j -= 1
  return i - j


# Finds the index where a "foo" or 'foo' string ends in the given string s. Given string s and index 'start' to a string symbol " or ', finds the matching index where the string ends.
# This takes into account escapes in the middle, i.e. "foo\\\\\\\"bar" will be properly matched.
def find_unescaped_end(s, ch, start, end):
  if s[start] != ch:
    raise Exception('Index start should point to starting occurrence of ch')
  start += 1
  while start < end:
    if s[start] == ch and rcount(s, '\\', start - 1) % 2 == 0:
      return start
    start += 1
  return -1


# Transforms linear index to string to file, column pair. (for debugging use only, need to build index->file:line mapping table for batch operations)
def idx_to_line_col(s, i):
  line = s.count('\n', 0, i) + 1
  last_n = s.rfind('\n', 0, i)
  return 'line ' + str(line) + ', column ' + str(i - last_n) + ' (idx ' + str(i) + ')'


# Given a string, returns brace_map dictionary that maps starting parens/brackets/braces indices to their ending positions.
# This can be brittle since we are not able to parse JS proper, but good enough for Emscripten compiled output. (some debugging code retained in body if you run into a tricky case)
def parse_parens(s):
  brace_map = {}

  parens = [] # ()
  brackets = [] # []
  braces = [] # {}

  i = 0
  end = len(s)
  while i < end:
    ch = s[i]
    if ch == '/':
      if i < end and s[i + 1] == '/':
        # prev = i
        i = s.find('\n', i)
        # print(idx_to_line_col(s, prev) + ' starts // comment, skipping to ' + idx_to_line_col(s, i))
      if i < end and s[i + 1] == '*':
        # prev = i
        i = s.find('*/', i + 2) + 1
        # print(idx_to_line_col(s, prev) + ' starts /* comment, skipping to ' + idx_to_line_col(s, i))
    elif ch == '"' and rcount(s, '\\', i - 1) % 2 == 0:
      # prev = i
      i = find_unescaped_end(s, '"', i, end)
      # print(idx_to_line_col(s, prev) + ' is a "" string, skipping to ' + idx_to_line_col(s, i))
    elif ch == "'" and rcount(s, '\\', i - 1) % 2 == 0:
      # prev = i
      i = find_unescaped_end(s, "'", i, end)
      # print(idx_to_line_col(s, prev) + ' is a \'\' string, skipping to ' + idx_to_line_col(s, i))
    elif ch == '^': # Ignore parens/brackets/braces if the previous character was a '^'. This is a bit of a heuristic, '^)' occur commonly in Emscripten generated regexes
      i += 1
    elif ch == '(':
      if rcount(s, '\\', i - 1) % 2 == 0:
        parens.append(i)
      # print(idx_to_line_col(s, i) + ' has (')
    elif ch == '[':
      if rcount(s, '\\', i - 1) % 2 == 0:
        brackets.append(i)
      # print(idx_to_line_col(s, i) + ' has [')
    elif ch == '{':
      if rcount(s, '\\', i - 1) % 2 == 0:
        braces.append(i)
      # print(idx_to_line_col(s, i) + ' has {')
    elif ch == ')':
      if rcount(s, '\\', i - 1) % 2 == 0:
        # print(idx_to_line_col(s, i) + ' has )')
        if len(parens) > 0:
          brace_map[parens.pop()] = i
        # else: print('Warning: ' + idx_to_line_col(s, i) + ' has ), but could not find the opening parenthesis.')
    elif ch == ']':
      if rcount(s, '\\', i - 1) % 2 == 0:
        # print(idx_to_line_col(s, i) + ' has ]')
        if len(brackets) > 0:
          brace_map[brackets.pop()] = i
        # else: print('Warning: ' + idx_to_line_col(s, i) + ' has ], but could not find the opening bracket.')
    elif ch == '}':
      if rcount(s, '\\', i - 1) % 2 == 0:
        # print(idx_to_line_col(s, i) + ' has }')
        if len(braces) > 0:
          brace_map[braces.pop()] = i
        # else: print('Warning: ' + idx_to_line_col(s, i) + ' has }, but could not find the opening brace.')
    i += 1
  return brace_map


# Valid characters in Emscripten outputted JS content (in reality valid character set is much more complex, but do not need that here)
def is_javascript_symbol_char(ch):
  i = ord(ch)
  return (i >= 97 and i <= 122) or (i >= 65 and i <= 90) or (i >= 48 and i <= 57) or i == 36 or i == 95 # a-z, A-Z, 0-9, $, _


def cxxfilt():
  filt = which('llvm-cxxfilt')
  if filt:
    return filt
  return which('c++filt')


# Runs the given symbols list through c++filt to demangle.
def cpp_demangle(symbols):
  try:
    filt = cxxfilt()
    if not filt:
      print('"llvm-cxxfilt" or "c++filt" executable is not found, demangled symbol names will not be available')
      return ''
    proc = subprocess.Popen([cxxfilt(), '--strip-underscore'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    output = proc.communicate(input=symbols)
    return output[0].replace('\r\n', '\n')
  except Exception:
    return ''


# Given a data set, fills in the 'demangled_data' field for each entry.
def find_demangled_names(data):
  if not data or len(data) == 0:
    return
  data_lines = list(data.keys())
  demangled_names = cpp_demangle('\n'.join(data_lines)).split('\n')
  for i in range(len(data)):
    mangled = data_lines[i]
    data[mangled]['demangled_name'] = demangled_names[i].strip() if i < len(demangled_names) else mangled


# Merges a new_entry with an old entry with the same name accumulating to its size (or adds new)
def merge_entry_to_existing(existing_data, new_entry, total_source_set_size):
  name = new_entry['unminified_name']
  if name in existing_data:
    ex = existing_data[name]
    num_times_occurs_1 = ex['num_times_occurs'] if 'num_times_occurs' in ex else 1
    num_times_occurs_2 = new_entry['num_times_occurs'] if 'num_times_occurs' in new_entry else 1
    existing_data[name] = {
      'lines': ex['lines'] + new_entry['lines'],
      'bytes': ex['bytes'] + new_entry['bytes'],
      'demangled_name': ex['demangled_name'] if 'demangled_name' in ex else (new_entry['demangled_name'] if 'demangled_name' in new_entry else new_entry['minified_name']),
      'minified_name': ex['minified_name'],
      'unminified_name': ex['unminified_name'],
      'function_parameters': ex['function_parameters'],
      'type': ex['type'],
      'percentage': (ex['bytes'] + new_entry['bytes']) * 100.0 / total_source_set_size,
      'num_times_occurs': num_times_occurs_1 + num_times_occurs_2
    }
  else:
    existing_data[name] = new_entry


def merge_to_data_set(to_set, from_set, total_source_set_size):
  for key, value in from_set.items():
    if diffing_two_data_sets:
      merge_entry_to_existing(to_set, value, total_source_set_size)
    else:
      # if key in to_set:
      #    key = s + '__' + key
      to_set[key] = value


# Builds up a dataset of functions and variables in the given JavaScript file (JS or asm.js)
def analyze_javascript_file_contents(filename, file_contents, total_source_set_size, symbol_map=None):
  data = {}
  brace_map = parse_parens(file_contents)
  parse_pos = 0
  prev_end_pos = 0
  file_len = len(file_contents)
  func_regex = re.compile(r'function\s+([\w$]+)\s*\(([\w\s$,]*?)\)\s*{') # Search for "function foo (param1, param2, ..., paranN) {"
  var_block_regex = re.compile(r'var\s+(\w+)\s*=\s*([{\[\(])') # Search for "var foo = {"
  var_regex = re.compile(r'var\s+([\w]+)\s*=\s*[\w\s,]*?;') # Search for "var foo = .... ;"
  unaccounted_bytes = 0
  unaccounted_lines = 0

  asm_start = file_contents.find('use asm')
  asm_start_brace = -1
  asm_end_brace = -1
  asm_type = 'asmjs'
  if asm_start < 0:
    asm_start = file_contents.find('almost asm')
    asm_type = '~asmjs'
  if asm_start >= 0:
    asm_start_brace = file_contents.rfind('{', 0, asm_start)
    if asm_start_brace >= 0:
      asm_end_brace = brace_map[asm_start_brace] if asm_start_brace in brace_map else file_len

  func_pos = -1
  var_pos = -1
  while parse_pos < file_len:
    if func_pos < parse_pos:
      func_pos = file_contents.find('function ', parse_pos)
    if func_pos < 0:
      func_pos = file_len
    if var_pos < parse_pos:
      var_pos = file_contents.find('var ', parse_pos)
    if var_pos < 0:
      var_pos = file_len
    if min(func_pos, var_pos) >= file_len:
      break
    next_pos = min(func_pos, var_pos)
    parse_pos = next_pos + 1

    # Skip this occurrence of 'function' if it had a prefix as part of some other string, e.g. 'foofunction'
    if next_pos > 0 and is_javascript_symbol_char(file_contents[next_pos - 1]):
      continue

    if next_pos > prev_end_pos:
      unaccounted_lines += file_contents.count('\n', prev_end_pos, next_pos) + 1
      unaccounted_bytes += next_pos - prev_end_pos
      if options.dump_unaccounted_larger_than >= 0 and next_pos - prev_end_pos > options.dump_unaccounted_larger_than:
        print('--- Unaccounted ' + str(next_pos - prev_end_pos) + ' bytes in ' + filename + ':')
        print(file_contents[prev_end_pos:next_pos])
        print('===')
    prev_end_pos = next_pos

    # Verify that this position actually starts a function by testing against a regex (this is much slower than substring search,
    # which is why it's done as a second step, instead of as primary way to search)
    if next_pos == func_pos:
      func_match = func_regex.match(file_contents[func_pos:])
      if not func_match:
        continue

      # find starting and ending braces { } for the function
      start_brace = file_contents.find('{', func_pos)
      if start_brace < 0:
        break # Must be at the end of file
      if start_brace not in brace_map:
        print('Warning: ' + idx_to_line_col(file_contents, start_brace) + ' cannot parse function start brace, skipping.')
        continue
      end_brace = brace_map[start_brace]
      if end_brace < 0:
        break # Must be at the end of file

      num_bytes = end_brace + 1 - func_pos
      num_lines = file_contents.count('\n', func_pos, end_brace) + 1
      prev_end_pos = parse_pos = end_brace + 1

      function_type = asm_type if func_pos >= asm_start_brace and end_brace <= asm_end_brace else 'js'
      minified_name = func_match.group(1)
      function_parameters = func_match.group(2).strip()
      if symbol_map and minified_name in symbol_map and function_type == asm_type:
        unminified_name = symbol_map[minified_name]
      else:
        unminified_name = minified_name
      data[unminified_name] = {
        'lines': num_lines,
        'bytes': num_bytes,
        'minified_name': minified_name,
        'unminified_name': unminified_name,
        'function_parameters': function_parameters,
        'type': function_type,
        'percentage': num_bytes * 100.0 / total_source_set_size
      }
    else: # This is a variable
      var_block_match = var_block_regex.match(file_contents[var_pos:])
      if var_block_match:
        # find starting and ending braces { } for the var
        start_brace = file_contents.find(var_block_match.group(2), var_pos)
        if start_brace < 0:
          break # Must be at the end of file
        if start_brace not in brace_map:
          print('Warning: ' + idx_to_line_col(file_contents, start_brace) + ' cannot parse variable start brace, skipping.')
          continue
        end_brace = brace_map[start_brace]
        if end_brace < 0:
          break # Must be at the end of file
        minified_name = var_block_match.group(1)
      else:
        start_brace = var_pos
        var_match = var_regex.match(file_contents[var_pos:])
        if not var_match:
          continue
        end_brace = file_contents.find(';', var_pos)
        minified_name = var_match.group(1)

      # Special case ignore the 'var asm = (function(global, env, buffer) { 'use asm'; ... }; ' variable that contains all the asm.js code.
      # Ignoring this variable lets all the asm.js code be trated as functions in this parser, instead of assigning them to the asm variable.
      if file_contents[start_brace] == '(' and ("'use asm'" in file_contents[var_pos:end_brace] or '"use asm"' in file_contents[var_pos:end_brace] or "'almost asm'" in file_contents[var_pos:end_brace] or '"almost asm"' in file_contents[var_pos:end_brace]):
        continue

      num_bytes = end_brace + 1 - var_pos
      num_lines = file_contents.count('\n', var_pos, end_brace) + 1
      prev_end_pos = parse_pos = end_brace + 1

      var_type = 'asm_var' if func_pos >= asm_start_brace and end_brace <= asm_end_brace else 'var'

      if symbol_map and minified_name in symbol_map and var_type == 'asm_var':
        unminified_name = symbol_map[minified_name].strip()
      else:
        unminified_name = minified_name
      data[unminified_name] = {
        'lines': num_lines,
        'bytes': num_bytes,
        'minified_name': minified_name,
        'unminified_name': unminified_name,
        'function_parameters': '',
        'type': var_type,
        'percentage': num_bytes * 100.0 / total_source_set_size
      }

  if options.list_unaccounted:
    if diffing_two_data_sets:
      unaccounted_name = '$unaccounted_js_content' # If diffing two data sets, must make the names of the unaccounted content blocks be comparable
    else:
      unaccounted_name = '$unaccounted_js_content_in("' + os.path.basename(filename) + '")'
    unaccounted_entry = {
      'lines': unaccounted_lines,
      'bytes': unaccounted_bytes,
      'minified_name': unaccounted_name,
      'unminified_name': unaccounted_name,
      'function_parameters': '',
      'type': '[UNKN]',
      'percentage': unaccounted_bytes * 100.0 / total_source_set_size
    }
    merge_entry_to_existing(data, unaccounted_entry, total_source_set_size)

  return data


def analyze_javascript_file(filename, total_source_set_size, symbol_map=None):
  file_contents = Path(filename).read_text()
  print('Analyzing JS file ' + filename + ', ' + str(len(file_contents)) + ' bytes...')
  return analyze_javascript_file_contents(filename, file_contents, total_source_set_size, symbol_map)


def analyze_html_file(filename, total_source_set_size, symbol_map=None):
  file_contents = Path(filename).read_text()
  print('Analyzing HTML file ' + filename + ', ' + str(len(file_contents)) + ' bytes...')
  data = {}
  parse_pos = 0
  file_len = len(file_contents)
  unaccounted_bytes = 0
  unaccounted_lines = 0

  while parse_pos < file_len:
    script_pos = file_contents.find('<script', parse_pos)
    if script_pos < 0:
      break
    script_pos = file_contents.find('>', script_pos)
    if script_pos < 0:
      break
    script_pos += 1
    script_end_pos = file_contents.find('</script>', script_pos)
    if script_end_pos < 0:
      break

    if script_pos > parse_pos:
      unaccounted_bytes += script_pos - parse_pos
      unaccounted_lines += file_contents.count('\n', parse_pos, script_pos) + 1
    data_set = analyze_javascript_file_contents(filename, file_contents[script_pos:script_end_pos], total_source_set_size, symbol_map)
    merge_to_data_set(data, data_set, total_source_set_size)
    parse_pos = script_end_pos

  if file_len > parse_pos:
    unaccounted_bytes += file_len - parse_pos
    unaccounted_lines += file_contents.count('\n', parse_pos, file_len) + 1

  if options.list_unaccounted and unaccounted_bytes > 0:
    if diffing_two_data_sets:
      unaccounted_name = '$unaccounted_html_content' # If diffing two data sets, must make the names of the unaccounted content blocks be comparable
    else:
      unaccounted_name = '$unaccounted_html_content_in("' + os.path.basename(filename) + '")'
    unaccounted_entry = {
      'lines': unaccounted_lines,
      'bytes': unaccounted_bytes,
      'minified_name': unaccounted_name,
      'unminified_name': unaccounted_name,
      'function_parameters': '',
      'type': 'HTML',
      'percentage': unaccounted_bytes * 100.0 / total_source_set_size
    }
    merge_entry_to_existing(data, unaccounted_entry, total_source_set_size)

  return data


def analyze_source_file(filename, total_source_set_size, symbol_map=None):
  if '.htm' in os.path.basename(filename).lower():
    return analyze_html_file(filename, total_source_set_size, symbol_map)
  else:
    return analyze_javascript_file(filename, total_source_set_size, symbol_map)


def common_compare(data1, data2):
  fns1 = set(data1.keys())
  fns2 = set(data2.keys())
  commonfns = fns1.intersection(fns2)
  commonlinediff = 0
  commonbytediff = 0
  for fn in commonfns:
    d1 = data1[fn]
    d2 = data2[fn]
    commonlinediff += d2['lines'] - d1['lines']
    commonbytediff += d2['bytes'] - d1['bytes']
  linesword = 'more' if commonlinediff >= 0 else 'less'
  bytesword = 'more' if commonbytediff >= 0 else 'less'
  print('set 2 has {} lines {} than set 1 in {} common functions'.format(abs(commonlinediff), linesword, len(commonfns)))
  print('set 2 has {} bytes {} than set 1 in {} common functions'.format(str(abs(commonbytediff)), bytesword, len(commonfns)))


def uniq_compare(data1, data2):
  fns1 = set(data1.keys())
  fns2 = set(data2.keys())
  uniqfns1 = fns1 - fns2
  uniqfns2 = fns2 - fns1
  uniqlines1 = 0
  uniqbytes1 = 0
  uniqlines2 = 0
  uniqbytes2 = 0
  for fn in uniqfns1:
    d = data1[fn]
    uniqlines1 += d['lines']
    uniqbytes1 += d['bytes']
  for fn in uniqfns2:
    d = data2[fn]
    uniqlines2 += d['lines']
    uniqbytes2 += d['bytes']
  uniqcountdiff = len(uniqfns2) - len(uniqfns1)
  assert len(fns2) - len(fns1) == uniqcountdiff
  uniqlinediff = uniqlines2 - uniqlines1
  uniqbytediff = uniqbytes2 - uniqbytes1
  countword = 'more' if uniqcountdiff >= 0 else 'less'
  linesword = 'more' if uniqlinediff >= 0 else 'less'
  bytesword = 'more' if uniqbytediff >= 0 else 'less'
  print('set 2 has {} functions {} than set 1 overall (unique: {} vs {})'.format(abs(uniqcountdiff), countword, len(uniqfns2), len(uniqfns1)))
  print('set 2 has {} lines {} than set 1 overall in unique functions'.format(abs(uniqlinediff), linesword))
  print('set 2 has {} bytes {} than set 1 overall in unique functions'.format(str(abs(uniqbytediff)), bytesword))


# Use a bunch of regexps to simplify the demangled name
DEM_RE = None


def simplify_cxx_name(name):
  global DEM_RE
  if DEM_RE is None:
    DEM_RE = []
    string_m = re.compile(r'std::__2::basic_string<char, std::__2::char_traits<char>, std::__2::allocator<char> >')
    DEM_RE.append(lambda s: string_m.sub(r'std::string', s))
    vec_m = re.compile(r'std::__2::vector<([^,]+), std::__2::allocator<\1\s*> >')
    DEM_RE.append(lambda s: vec_m.sub(r'std::vector<\1>', s))
    unordered_map_m = re.compile(r'std::__2::unordered_map<([^,]+), ([^,]+), std::__2::hash<\1\s*>, std::__2::equal_to<\1\s*>, std::__2::allocator<std::__2::pair<\1 const, \2> > >')
    DEM_RE.append(lambda s: unordered_map_m.sub(r'std::unordered_map<\1, \2>', s))
    sort_m = re.compile(r'std::__2::__sort<std::__2::__less<([^,]+), \1\s*>&, \1\*>\(\1\*, \1\*, std::__2::__less<\1, \1\s*>&\)')
    DEM_RE.append(lambda s: sort_m.sub(r'std::sort(\1*, \1*)', s))
    DEM_RE.append(lambda s: s.replace('std::__2::', 'std::'))

  for dem in DEM_RE:
    name = dem(name)
  return name


# 'foo(int, float)' -> 'foo'
def function_args_removed(s):
  if '(' in s:
    return s[:s.find('(')]
  else:
    return s


# 'foo(int, float)' -> 'int, float)'
def function_args_part(s):
  if '(' in s:
    return s[s.find('(') + 1:]
  else:
    return ''


def sort_key_py2(key_value):
  return key_value[1][options.sort]


# Apparently for python 3, one will use the following, but currently untested
# def sort_key_py3(key, value):
#   return value[options.sort]

def print_symbol_info(data, total_source_set_size):
  data = list(data.items())
  data.sort(key=sort_key_py2, reverse=not options.sort_ascending)

  total_size = 0
  for unminified_name, e in data:
    if options.only_unique_1 and e['in_set_2']:
      continue
    if options.only_unique_2 and e['in_set_1']:
      continue
    if options.only_common and (not e['in_set_1'] or not e['in_set_2']):
      continue
    prev_bytes = e['prev_bytes'] if 'prev_bytes' in e else 0
    if max(e['bytes'], prev_bytes) < options.filter_size:
      continue
    if e['bytes'] == prev_bytes and options.only_changes:
      continue

    minified_name = e['minified_name']
    demangled_name = e['demangled_name']
    if options.simplify_cxx:
      demangled_name = simplify_cxx_name(demangled_name)

    if '(' not in demangled_name and 'js' in e['type']:
      demangled_name_with_args = demangled_name + '(' + e['function_parameters'] + ')'
    else:
      demangled_name_with_args = demangled_name
    demangled_name = function_args_removed(demangled_name)

    if options.filter_name not in demangled_name_with_args.lower():
      continue

    if e['function_parameters']:
      unminified_name_with_args = unminified_name + '(' + e['function_parameters'] + ')'
      minified_name_with_args = minified_name + '(' + e['function_parameters'] + ')'
    elif 'js' in e['type']:
      unminified_name_with_args = unminified_name + '()'
      minified_name_with_args = minified_name + '()'
    else:
      unminified_name_with_args = unminified_name
      minified_name_with_args = minified_name

    # Build up the function name to print based on the desired formatting specifiers (mangled/minified/unminified, yes/no args)
    print_name = []
    for i in options.print_format:
      if i == 'd':
        print_name += [demangled_name]
      elif i == 'u':
        print_name += [unminified_name]
      elif i == 'm':
        print_name += [minified_name]
      elif i == 'D':
        print_name += [demangled_name_with_args]
      elif i == 'U':
        print_name += [unminified_name_with_args]
      elif i == 'M':
        print_name += [minified_name_with_args]

    # Collapse names that are identical
    i = 0
    while i + 1 < len(print_name):
      if print_name[i] == print_name[i + 1]:
        print_name = print_name[:i] + print_name[i + 1:]
        continue
      n1 = function_args_removed(print_name[i])
      n2 = function_args_removed(print_name[i + 1])
      args1 = function_args_part(print_name[i])
      args2 = function_args_part(print_name[i + 1])
      if n1 == n2 and (not args1 or not args2):
        if not args1:
          print_name = print_name[:i] + print_name[i + 1:]
        else:
          print_name = print_name[:i + 1] + print_name[i + 2:]
        continue
      i += 1

    print_name = ' ; '.join(print_name)
    if 'num_times_occurs' in e:
      print_name = '[' + str(e['num_times_occurs']) + ' times] ' + print_name
    delta_string = ' %+8d (%+6.2f%%)' % (e['bytes'] - e['prev_bytes'], e['percentage'] - e['prev_percentage']) if diffing_two_data_sets else ''
    print('%6d lines %7s (%5.2f%%) %s: %8s %s' % (e['lines'], str(e['bytes']), e['percentage'], delta_string, e['type'], print_name))

    total_size += e['bytes']

  if total_size < total_source_set_size:
    print('Total size of printed functions: ' + str(total_size) + ' bytes. (%.2f%% of all symbols)' % (total_size * 100.0 / total_source_set_size))
  else:
    print('Total size of printed functions: ' + str(total_size) + ' bytes.')


# Parses Emscripten compiler generated .symbols map file for minified->unminified mappings
def read_symbol_map(filename):
  if not filename:
    return
  symbol_map = {}
  for line in open(filename):
    minified, unminified = line.split(':')
    symbol_map[minified.strip()] = unminified.strip()
  return symbol_map


# Locates foo.js to foo.js.symbols or foo.html.symbols based on default output name rules for Emscripten compiler
def guess_symbol_map_file_location(sources, symbol_map_file):
  if os.path.isfile(symbol_map_file):
    return symbol_map_file
  for s in sources:
    if os.path.isfile(s + '.symbols'):
      return s + '.symbols'
    if os.path.isfile(s.replace('.js', '.html') + '.symbols'):
      return s.replace('.js', '.html') + '.symbols'
  return None


# Returns total byte size of the given list of source files
def count_file_set_size(sources):
  total_size = 0
  for s in sources:
    total_size += os.path.getsize(s)
  return total_size


# Merges two given data sets into one large data set with diffing information
def diff_data_sets(data1, data2):
  all_keys = set().union(data1.keys(), data2.keys())
  diffed_data = {}
  for k in all_keys:
    if k in data2:
      e = data2[k].copy()
      e['in_set_2'] = True
      if k in data1:
        prev = data1[k]
        e['prev_percentage'] = prev['percentage']
        e['prev_bytes'] = prev['bytes']
        e['prev_lines'] = prev['lines']
        e['in_set_1'] = True
      else:
        e['prev_percentage'] = 0
        e['prev_bytes'] = 0
        e['prev_lines'] = 0
        e['in_set_1'] = False
    else:
      e = data1[k].copy()
      e['prev_percentage'] = e['percentage']
      e['prev_lines'] = e['lines']
      e['prev_bytes'] = e['bytes']
      e['in_set_1'] = True
      if k in data2:
        e['percentage'] = prev['percentage']
        e['bytes'] = prev['bytes']
        e['lines'] = prev['lines']
        e['in_set_2'] = True
      else:
        e['percentage'] = 0
        e['bytes'] = 0
        e['lines'] = 0
        e['in_set_2'] = False
    e['delta'] = e['bytes'] - e['prev_bytes']
    e['delta_percentage'] = e['percentage'] - e['prev_percentage']
    e['abs_delta'] = abs(e['bytes'] - e['prev_bytes'])
    diffed_data[k] = e
  return diffed_data


# Given string s and start index that contains a (, {, <, [, ", or ', finds forward the index where the token closes (taking nesting into account)
def find_index_of_closing_token(s, start):
  start_ch = s[start]
  if start_ch == '(':
    end_ch = ')'
  elif start_ch == '{':
    end_ch = '}'
  elif start_ch == '<':
    end_ch = '>'
  elif start_ch == '[':
    end_ch = ']'
  elif start_ch == '"':
    end_ch = '"'
  elif start_ch == "'":
    end_ch = "'"
  else:
    raise Exception('Unknown start token ' + start_ch + ', string ' + s + ', start ' + start)

  i = start + 1
  nesting_count = 1
  while i < len(s):
    if s[i] == end_ch:
      nesting_count -= 1
      if nesting_count <= 0:
        return i
    elif s[i] == start_ch:
      nesting_count += 1
    i += 1
  return i


def compute_templates_collapsed_name(demangled_name):
  i = 0
  generic_template_name = 'T'
  type_names = {}
  while True:
    i = demangled_name.find('<', i)
    if i < 0:
      return demangled_name

    end = find_index_of_closing_token(demangled_name, i)
    if end < 0:
      return demangled_name

    i += 1
    template_type = demangled_name[i:end]
    if template_type in type_names:
      template_name = type_names[template_type]
    else:
      template_name = generic_template_name
      type_names[template_type] = generic_template_name
      generic_template_name = chr(ord(generic_template_name) + 1)

    demangled_name = demangled_name[:i] + template_name + demangled_name[end:]


def collapse_templates(data_set, total_source_set_size, no_function_args):
  collapsed_data_set = {}
  keys = data_set.keys()
  for k in keys:
    e = data_set[k]
    if 'demangled_name' in e:
      demangled_name = compute_templates_collapsed_name(e['demangled_name'])
      if no_function_args:
        demangled_name = function_args_removed(demangled_name)
      e['demangled_name'] = e['unminified_name'] = demangled_name
    merge_entry_to_existing(collapsed_data_set, e, total_source_set_size)
  return collapsed_data_set


def print_function_args(options):
  return 'D' in options.print_format or 'U' in options.print_format or 'M' in options.print_format


def main():
  global options, diffing_two_data_sets
  usage_str = "emdump.py prints out statistics about compiled code sizes.\npython emdump.py --file a.js [--file2 b.js]"
  parser = argparse.ArgumentParser(usage=usage_str)

  parser.add_argument('--file', dest='file', default=[], nargs='*',
                      help='Specifies the compiled JavaScript build file to analyze.')

  parser.add_argument('--file1', dest='file1', default=[], nargs='*',
                      help='Specifies the compiled JavaScript build file to analyze.')

  parser.add_argument('--symbol-map', dest='symbol_map', default='',
                      help='Specifies a filename to the symbol map file that can be used to unminify function and variable names.')

  parser.add_argument('--file2', dest='file2', default=[], nargs='*',
                      help='Specifies a second compiled JavaScript build file to analyze.')

  parser.add_argument('--symbol-map2', dest='symbol_map2', default='',
                      help='Specifies a filename to a second symbol map file that will be used to unminify function and variable names of file2.')

  parser.add_argument('--list-unaccounted', dest='list_unaccounted', type=int, default=1,
                      help='Pass --list-unaccounted=0 to skip listing a summary entry of unaccounted content')

  parser.add_argument('--dump-unaccounted-larger-than', dest='dump_unaccounted_larger_than', type=int, default=-1,
                      help='If an integer value >= 0 is specified, all unaccounted strings of content longer than the given value will be printed out to the console.\n(Note that it is common to have several unaccounted blocks, this is provided for curiosity/debugging/optimization ideas)')

  parser.add_argument('--only-unique-1', dest='only_unique_1', action='store_true', default=False,
                      help='If two data sets are specified, prints out only the symbols that are present in set 1, but not in set 2')

  parser.add_argument('--only-unique-2', dest='only_unique_2', action='store_true', default=False,
                      help='If two data sets are specified, prints out only the symbols that are present in set 2, but not in set 1')

  parser.add_argument('--only-common', dest='only_common', action='store_true', default=False,
                      help='If two data sets are specified, prints out only the symbols that are common to both data sets')

  parser.add_argument('--only-changes', dest='only_changes', action='store_true', default=False,
                      help='If two data sets are specified, prints out only the symbols that have changed size or are added/removed')

  parser.add_argument('--only-summarize', dest='only_summarize', action='store_true', default=False,
                      help='If specified, detailed information about each symbol is not printed, but only summary data is shown.')

  parser.add_argument('--filter-name', dest='filter_name', default='',
                      help='Only prints out information about symbols that contain the given filter substring in their demangled names. The filtering is always performed in lower case.')

  parser.add_argument('--filter-size', dest='filter_size', type=int, default=0,
                      help='Only prints out information about symbols that are (or were) larger than the given amount of bytes.')

  parser.add_argument('--sort', dest='sort', default='bytes',
                      help='Specifies the data column to sort output by. Possible values are: lines, bytes, delta, abs_delta, type, minified, unminified, demangled')

  parser.add_argument('--print-format', dest='print_format', default='DM',
                      help='Specifies the naming format for the symbols. Possible options are one of: m, u, d, du, dm, um, dum. Here "m" denotes minified, "u" denotes unminified, and "d" denotes demangled. Specify any combination of the characters in upper case to print out function parameters.\nDefault: DM.')

  parser.add_argument('--sort-ascending', dest='sort_ascending', action='store_true', default=False,
                      help='If true, reverses the sorting order to be ascending instead of default descending.')

  parser.add_argument('--simplify-cxx', dest='simplify_cxx', action='store_true', default=False,
                      help='Simplify C++ STL types as much as possible in the output')

  parser.add_argument('--group-templates', dest='group_templates', action='store_true', default=False,
                      help='Group/collapse all C++ templates with Foo<asdf> and Foo<qwer> to generic Foo<T>')

  options = parser.parse_args()
  options.file = options.file + options.file1

  if not options.file:
    print('Specify a set of JavaScript build output files to analyze with --file file1.js file2.js ... fileN.js.\nRun python emdump.py --help to see all options.')
    return 1

  options.filter_name = options.filter_name.lower()

  diffing_two_data_sets = len(options.file2) > 0
  if not diffing_two_data_sets:
    if options.only_unique_1:
      print('Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --only-unique-symbols-in-set-1 option!')
      sys.exit(1)

    if options.only_unique_2:
      print('Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --only-unique-symbols-in-set-2 option!')
      sys.exit(1)

    if options.only_common:
      print('Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --only-common-symbols option!')
      sys.exit(1)

  # Validate column sorting input:
  valid_sort_options = ['lines', 'bytes', 'delta', 'abs_delta', 'type', 'minified', 'unminified', 'demangled']
  if options.sort not in valid_sort_options:
    print('Invalid sort option ' + options.sort + ' specified! Choose one of: ' + ', '.join(valid_sort_options) + '.')
    sys.exit(1)
  if options.sort == 'minified':
    options.sort = 'minified_name'
  if options.sort == 'unminified':
    options.sort = 'unminified_name'
  if options.sort == 'demangled':
    options.sort = 'demangled_name'

  if 'delta' in options.sort and not diffing_two_data_sets:
    print('Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --sort=' + options.sort)
    sys.exit(1)

  # Autoguess .symbols file location based on default Emscripten build output, to save the need to type it out in the common case
  options.symbol_map = guess_symbol_map_file_location(options.file, options.symbol_map)
  options.symbol_map2 = guess_symbol_map_file_location(options.file2, options.symbol_map2)

  symbol_map1 = read_symbol_map(options.symbol_map)
  symbol_map2 = read_symbol_map(options.symbol_map2)

  set1_size = count_file_set_size(options.file)
  data1 = {}
  for s in options.file:
    data = analyze_source_file(s, set1_size, symbol_map1)
    merge_to_data_set(data1, data, set1_size)

  set2_size = count_file_set_size(options.file2)
  data2 = {}
  for s in options.file2:
    data = analyze_source_file(s, set2_size, symbol_map2)
    merge_to_data_set(data2, data, set2_size)

  find_demangled_names(data1)
  find_demangled_names(data2)

  if options.group_templates:
    data1 = collapse_templates(data1, set1_size, not print_function_args(options))
    data2 = collapse_templates(data2, set2_size, not print_function_args(options))

  if diffing_two_data_sets:
    diffed_data = diff_data_sets(data1, data2)
    if not options.only_summarize:
      print_symbol_info(diffed_data, set2_size)
      print('')
    print('set 2 is %d bytes, which is %+.2f%% %s than set 1 size (%d bytes)' % (set2_size, (set2_size - set1_size) * 100.0 / set2_size, 'more' if set2_size > set1_size else 'less', set1_size))
    uniq_compare(data1, data2)
    common_compare(data1, data2)
  else:
    if not options.only_summarize:
      print_symbol_info(data1, set1_size)
    # TODO: print some kind of summary?

  return 0


if __name__ == '__main__':
  sys.exit(main())